爬虫技术（四）-- 简单爬虫抓取示例（附c#代码）

using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Data;
using System.Drawing;
using System.Linq;
using System.Text;
using System.Windows.Forms;
using System.Web.Security;
using System.IO;
using System.Net;
using System.Text.RegularExpressions;
using System.Web;

namespace Demo1
{

    public partial class Form1 : Form
    {
        public Form1()
        {
            InitializeComponent();
         }

       
        private void button1_Click(object sender, EventArgs e)
        {
            Test1 a = new Test1();
            a.getCurrentURL();
        }

        public class Test1
        {
            List<string> todo = new List<string>();
            List<string> visited = new List<string>();
            string startPoint = "http://www.cnblogs.com/lmei/";
            
            public void getCurrentURL()
            {
                RequestSite(startPoint);

                while (todo.Count > 0) 
                {
                    string currentURL = todo[0];
                    
                    RequestSite(currentURL);

                    if (visited.Contains(currentURL))  //注释1
                    {
                        Console.WriteLine("已经访问过了" + currentURL);
                        todo.Remove((currentURL));
                    }
                    else
                    {
                        Console.WriteLine("现在正在访问：===>   " + currentURL);
                        visited.Add(currentURL);

                        Console.WriteLine("目前已经访问了：===>   " + visited.Count + "个网页" );
                        todo.Remove((currentURL));
                    }
                }
            }

            public void RequestSite(string url)
            {
                WebRequest req = WebRequest.Create(url);
                HttpWebResponse res;
                try{
                      res = (HttpWebResponse)(req.GetResponse());
                }
                catch (WebException ex) { res = (HttpWebResponse)ex.Response; }

                Stream st = res.GetResponseStream();
                StreamReader rdr = new StreamReader(st);
                string s = rdr.ReadToEnd();
                todo.AddRange(GetLink(s));
            }

            List<string> GetLink(string htmlPage)
            {
                
                Regex regx =
                    new Regex("http://www\\.cnblogs\\.com\\/lmei\\/p\\/[0-9a-zA-Z]+\\.html*" ,RegexOptions.IgnoreCase);
                MatchCollection matches = regx.Matches(htmlPage);

                List<string> results = new List<string>();
                foreach (Match match in matches)
                {
                    if (!visited.Contains(match.Value)) //注释2
                    {
                        results.Add(match.Value);
                    }
                }
                return results;
            }
        }
      
    }
}