August 14, 2020 C# basic knowledge crawler

#regionGrab Tencent class category data
ISearch search = new CategorySearch();
search.Crawler();
#endregion
#regionFetching course
ISearch search1 = new CourseSearch(category);
search1.Crawler();
#endregion
#regionGetting Ajax data
CourseSearch courseSearch = new CourseSearch();
courseSearch.GetAjaxRequest();
#endregion
Copy the code

CategorySearch – Crawler

/// <summary>
    ///http://www.w3school.com.cn/xpath/index.asp XPATH syntax
    /// </summary>
    public class CategorySearch : ISearch
    {
        private static Logger logger = new Logger(typeof(CategorySearch));
        private int _Count = 1;// Each time a new class is reinitialized


        /// <summary>
        ///If the crawler needs to obtain all the course data of Tencent classroom, it needs to obtain it through categories
        /// 
        ///Or request to obtain Html content parsing filtering information, obtain effective information into the library
        /// </summary>
        public void Crawler()
        {
            List<TencentCategoryEntity> categoryList = new List<TencentCategoryEntity>();
            try
            {
                // Configure the url to crawl
                string url = $"{Constant.TencentClassUrl}/course/list/? tuin=7e4f8b7d";
                // Load the HTML in the URL
                string html = HttpHelper.DownloadUrl(url);
                // HTML document parsing
                HtmlDocument doc = new HtmlDocument();
                doc.LoadHtml(html);
                // Parse the xpath to get the corresponding node
                string fristPath = "//*[@id=\"auto-test-1\"]/div[1]/dl/dd";
                HtmlNodeCollection nodeList = doc.DocumentNode.SelectNodes(fristPath);
                if (nodeList == null) {}foreach (HtmlNode node in nodeList)
                {
                    categoryList.AddRange(this.First(node.InnerHtml, null));
                }
                // Import the content into the database (the page data has been obtained before, can be stored in the corresponding database)
                CategoryRepository categoryRepository = new CategoryRepository();
                categoryRepository.Save(categoryList);
            }
            catch (Exception ex)
            {
                logger.Error("CrawlerMuti is abnormal.", ex);
            }
            finally
            {
                Console.WriteLine($" type data initialization completed, total fetching category{ categoryList? .Count}A"); }}/// <summary>
        ///Look up each of the first-level classes
        /// </summary>
        /// <param name="html"></param>
        /// <param name="code"></param>
        /// <param name="parentCode"></param>
        /// <returns></returns>
        private List<TencentCategoryEntity> First(string html, string parentCode)
        {
            List<TencentCategoryEntity> categoryList = new List<TencentCategoryEntity>();
            HtmlDocument doc = new HtmlDocument();
            doc.LoadHtml(html);
            string namePath = "//a/h2";
            HtmlNode name = doc.DocumentNode.SelectSingleNode(namePath);
            string codePath = "//a";
            HtmlNode codeNode = doc.DocumentNode.SelectSingleNode(codePath);
            string href = codeNode.Attributes["href"].Value;

            string code = string.Empty;
            if(href ! =null && href.IndexOf("mt=") != - 1)
            {
                href = href.Replace(";"."&");
                code = href.Substring(href.IndexOf("mt=") + 3.4);
            }
            TencentCategoryEntity category = new TencentCategoryEntity()
            {
                Id = _Count++,
                State = 1,
                CategoryLevel = 1,
                Code = code,
                ParentCode = parentCode
            };
            category.Name = name.InnerText;
            category.Url = href;
            categoryList.Add(category);
            if(name.InnerText ! ="All")
            {
                categoryList.AddRange(this.Second($"{Constant.TencentClassUrl}{href}&tuin=7e4f8b7d", code));
            }
            return categoryList;
        }

        /// <summary>
        ///Find all secondary classes below a primary class
        /// </summary>
        /// <param name="html"></param>
        /// <param name="parentCode"></param>
        /// <returns></returns>
        private List<TencentCategoryEntity> Second(string url, string parentCode)
        {
            string html = HttpHelper.DownloadUrl(url);
            List<TencentCategoryEntity> categoryList = new List<TencentCategoryEntity>();
            HtmlDocument doc = new HtmlDocument();
            doc.LoadHtml(html);
            string path = "//*[@id='auto-test-1']/div[1]/dl/dd";
            HtmlNodeCollection nodeList = doc.DocumentNode.SelectNodes(path);

            foreach (HtmlNode node in nodeList)
            {
                HtmlDocument htmlDocument = new HtmlDocument();
                htmlDocument.LoadHtml(node.InnerHtml);

                string codePath = "//a";
                HtmlNode codeNode = htmlDocument.DocumentNode.SelectSingleNode(codePath);
                string href = codeNode.Attributes["href"].Value;
                if (!string.IsNullOrWhiteSpace(href))
                {
                    href = href.Replace(";"."&");
                }

                string code = string.Empty;
                if(href ! =null && href.IndexOf("st=") != - 1)
                {
                    href = href.Replace(";"."&");
                    code = href.Substring(href.IndexOf("st=") + 3.4);
                }
                TencentCategoryEntity category = new TencentCategoryEntity()
                {
                    Id = _Count++,
                    State = 1,
                    CategoryLevel = 2,
                    Code = code,
                    ParentCode = parentCode
                };
                category.Name = codeNode.InnerText;
                category.Url = href;

                categoryList.Add(category);

                if(codeNode.InnerText ! ="All")
                {
                    categoryList.AddRange(this.Third($"{Constant.TencentClassUrl}{href}&tuin=7e4f8b7d", code)); }}return categoryList;
        }

        /// <summary>
        ///Find all three classes in a second class
        /// </summary>
        /// <param name="html"></param>
        /// <param name="parentCode"></param>
        /// <returns></returns>
        private List<TencentCategoryEntity> Third(string url, string parentCode)
        {
            string html = HttpHelper.DownloadUrl(url);
            List<TencentCategoryEntity> categoryList = new List<TencentCategoryEntity>();
            HtmlDocument doc = new HtmlDocument();
            doc.LoadHtml(html);
            string path = "//*[@id='auto-test-1']/div[1]/dl/dd";
            HtmlNodeCollection nodeList = doc.DocumentNode.SelectNodes(path);
            if (nodeList == null) {}foreach (HtmlNode node in nodeList)
            {
                HtmlDocument htmlDocument = new HtmlDocument();

                htmlDocument.LoadHtml(node.InnerHtml);

                string codePath = "//a";
                HtmlNode codeNode = htmlDocument.DocumentNode.SelectSingleNode(codePath);
                string href = codeNode.Attributes["href"].Value;

                string code = string.Empty;
                if(href ! =null)
                {
                    href = href.Replace(";"."&");
                }
                if(href ! =null && href.IndexOf("tt=") != - 1)
                {
                    href = href.Replace(";"."&");
                    code = href.Substring(href.IndexOf("tt=") + 3.4);
                }
                TencentCategoryEntity category = new TencentCategoryEntity()
                {
                    Id = _Count++,
                    State = 1,
                    CategoryLevel = 3,
                    Code = code,
                    ParentCode = parentCode
                };
                category.Name = codeNode.InnerText;
                category.Url = href;
                categoryList.Add(category);
            }
            returncategoryList; }}Copy the code

public class HttpHelper
{
    private static Logger logger = new Logger(typeof(HttpHelper));

    /// <summary>
    ///GB2312 before downloading the content according to the URL
    /// </summary>
    /// <param name="url"></param>
    /// <returns></returns>
    public static string DownloadUrl(string url)
    {
        return DownloadHtml(url, Encoding.UTF8);
    }

    //HttpClient--WebApi

    /// <summary>
    ///Download the HTML
    /// http://tool.sufeinet.com/HttpHelper.aspx
    ///HttpWebRequest features are relatively rich, WebClient is relatively simple to use
    /// WebRequest
    /// 
    /// </summary>
    /// <param name="url"></param>
    /// <returns></returns>
    public static string DownloadHtml(string url, Encoding encode)
    {
        string html = string.Empty;
        try
        {
            HttpWebRequest request = HttpWebRequest.Create(url) as HttpWebRequest;// simulate the request
            request.Timeout = 30 * 1000;// Set the timeout to 30 seconds
            request.UserAgent = "Mozilla / 5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.106 Safari/537.36";
            request.ContentType = "text/html; charset=utf-8";// "text/html; charset=gbk"; //
            //request.Host = "search.yhd.com";
            //request.Headers.Add("Cookie", @"newUserFlag=1; guid=YFT7C9E6TMFU93FKFVEN7TEA5HTCF5DQ26HZ; gray=959782; cid=av9kKvNkAPJ10JGqM_rB_vDhKxKM62PfyjkB4kdFgFY5y5VO; abtest=31; _ga = GA1.2.334889819.1425524072; grouponAreaId=37; provinceId=20; search_showFreeShipping=1; RURL=http%3A%2F%2Fsearch.yhd.com % 2 2 fkiphone fc0-0% % 2 f20%2 f % 3 FTP % 3 d1. 1.12.0.73. Ko3mjRR - 11 - FH7eo; aut=5GTM45VFJZ3RCTU21MHT4YCG1QTYXERWBBUFS4; ac=57265177%40qq.com; msessionid=H5ACCUBNPHMJY3HCK4DRF5VD5VA9MYQW; gc=84358431%2C102362736%2C20001585%2C73387122; The tma = 40580330.95741028.1425524063040.1430288358914.1430790348439.9; TMD = 23.40580330.95741028.1425524063040. search_browse_history=998435%2C1092925%2C32116683%2C1013204%2C6486125%2C38022757%2C36224528%2C24281304%2C22691497%2C2602 9325; detail_yhdareas=""; cart_cookie_uuid=b64b04b6-fca7-423b-b2d1-ff091d17e5e5; Gla _0_0 = 20.237; JSESSIONID=14F1F4D714C4EE1DD9E11D11DDCD8EBA; wide_screen=1; linkPosition=search");
            //request.Headers.Add("Accept", "text/html,application/xhtml+xml,application/xml; Q = 0.9, image/webp, * / *; Q = 0.8 ");
            //request.Headers.Add("Accept-Encoding", "gzip, deflate, sdch");
            //request.Headers.Add("Referer", "http://list.yhd.com/c0-0/b/a-s1-v0-p1-price-d0-f0-m1-rt0-pid-mid0-kiphone/");
            //Encoding enc = Encoding.GetEncoding("GB2312"); // UtF-8 / GB2312
            // How to automatically read cookies
            request.CookieContainer = new CookieContainer();//1 Prepare a container for the request
            using (HttpWebResponse response = request.GetResponse() as HttpWebResponse)// Initiate a request
            {
                if(response.StatusCode ! = HttpStatusCode.OK) { logger.Warn(string.Format("Failed to fetch {0} address,response.StatusCode is {1}", url, response.StatusCode));
                }
                else
                {
                    try
                    {
                        //string sessionValue = response.Cookies["ASP.NET_SessionId"].Value; / / 2 read cookies
                        StreamReader sr = new StreamReader(response.GetResponseStream(), encode);
                        html = sr.ReadToEnd();// Read data
                        sr.Close();
                    }
                    catch (Exception ex)
                    {
                        logger.Error(string.Format($" DownloadHtml fetching{url}Failure"), ex);
                        html = null;
                    }
                }
            }
        }
        catch (System.Net.WebException ex)
        {
            if (ex.Message.Equals("The remote server returned an error: (306)."))
            {
                logger.Error("The remote server returned an error: (306).", ex);
                html = null;
            }
        }
        catch (Exception ex)
        {
            logger.Error(string.Format("DownloadHtml fetch {0} error", url), ex);
            html = null;
        }
        returnhtml; }}Copy the code

CourseSearch – Crawler

/// <summary>
    ///Goods to grab
    ///http://www.w3school.com.cn/xpath/index.asp XPATH syntax
    /// 
    ///1 HtmlAgilityPack is pretty handy
    ///2 customized, different sites to customize;
    ///The same site basically does not need to be upgraded
    /// </summary>
    public class CourseSearch : ISearch
    {
        private Logger logger = new Logger(typeof(CourseSearch));
        private WarnRepository warnRepository = new WarnRepository();
        private CourseRepository courseRepository = new CourseRepository();
        private TencentCategoryEntity category = null;

        public CourseSearch(){}public CourseSearch(TencentCategoryEntity _category)
        {
            category = _category;
        }

        public void Crawler()
        {
            try
            {
                if (string.IsNullOrEmpty(category.Url))
                {
                    warnRepository.SaveWarn(category, string.Format(Name={0} Level={1} Url={2}", category.Name, category.CategoryLevel, category.Url));
                    return;
                }
                { 
                    #regionPaging to obtain
                    //ImageHelper.DeleteDir(Constant.ImagePath);
                   GetPageCourseData();
                    #endregion

                    #regionGet data for a page
                    //this.Show(category.Url);
                    #endregion
                }
            }
            catch (Exception ex)
            {
                logger.Error("CrawlerMuti is abnormal.", ex);
                warnRepository.SaveWarn(category, string.Format(Name={0} Level={1} Url={2}, category.Name, category.CategoryLevel, category.Url)); }}static int count = 0;

        // This crawler custom routines if you can understand; Brush a 1
        public void Show(string url)
        {
            string strHtml = HttpHelper.DownloadUrl(url);
            HtmlDocument document = new HtmlDocument();
            document.LoadHtml(strHtml);
            string liPath = "/html/body/section[1]/div/div[@class='market-bd market-bd-6 course-list course-card-list-multi-wrap js-course-list']/ul/li";
            HtmlNodeCollection liNodes = document.DocumentNode.SelectNodes(liPath);
            foreach (var node in liNodes)
            {
                Console.WriteLine();
                Console.WriteLine();
                Console.WriteLine("* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *");
                HtmlDocument lidocument = new HtmlDocument();
                lidocument.LoadHtml(node.OuterHtml);
                string aPath = "//*/a[1]";
                HtmlNode classANode = lidocument.DocumentNode.SelectSingleNode(aPath);
                string aHref = classANode.Attributes["href"].Value;

                Console.WriteLine($" course Url:{aHref}");

                string Id = classANode.Attributes["data-id"].Value;

                Console.WriteLine($" course Id:{Id}");

                string imgPath = "//*/a[1]/img";
                HtmlNode imgNode = lidocument.DocumentNode.SelectSingleNode(imgPath);
                string imgUrl = imgNode.Attributes["src"].Value;

                Console.WriteLine($"ImageUrl:{imgUrl}");

                string namePaths = "//*/h4/a[1]";
                HtmlNode nameNode = lidocument.DocumentNode.SelectSingleNode(namePaths);
                string name = nameNode.InnerText;
                Console.WriteLine(name);

                Console.WriteLine(Course Name:{name}");
                // courseEntity.Price = new Random().Next(100, 10000); // This is an advanced content that cannot be done by ordinary means (he has his own algorithm)

                count = count + 1; }}public void ShowPageData(string url)
        {
            string strHtml = HttpHelper.DownloadUrl(url);
            HtmlDocument document = new HtmlDocument();
            document.LoadHtml(strHtml);
            string pagePath = "/html/body/section[1]/div/div[5]/a[@class='page-btn']";
            HtmlNodeCollection pageNodes = document.DocumentNode.SelectNodes(pagePath);
            int maxPage = pageNodes.Select(p => int.Parse(p.InnerText)).Max();
            for (int page = 1; page <= maxPage; page++)
            {
                string pageUrl = $"{url}&page={page}";
                Show(pageUrl);
            }
            Console.WriteLine($" Total fetch data{count}Article");
        }

        #regionPaging grab
        private void GetPageCourseData()
        {
            //1. Determine the total page count
            //2. Fetch the data of each page separately
            //3. Analysis, filtration and cleaning
            / / 4. Put in storage

            category.Url = $"{Constant.TencentClassUrl}{category.Url}";

            string strHtml = HttpHelper.DownloadUrl(category.Url);
            HtmlDocument document = new HtmlDocument();
            document.LoadHtml(strHtml);
            //Xpath
            string pagePath = "/html/body/section[1]/div/div[@class='sort-page']/a[@class='page-btn']";
            HtmlNodeCollection pageNodes = document.DocumentNode.SelectNodes(pagePath);

            int pageCount = 1;
            if(pageNodes ! =null)
            {
                pageCount = pageNodes.Select(a => int.Parse(a.InnerText)).Max();
            }
            List<CourseEntity> courseList = new List<CourseEntity>();

            for (int pageIndex = 1; pageIndex <= pageCount; pageIndex++)
            {
                Console.WriteLine($"****************************** is currently the no{pageIndex}Data on page * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *");
                string pageIndexUrl = $"{category.Url}&page={pageIndex}";
                List<CourseEntity> courseEntities = GetPageIndeData(pageIndexUrl);
                courseList.AddRange(courseEntities);
            }
            courseRepository.SaveList(courseList);


        }

        private List<CourseEntity> GetPageIndeData(string url)
        {
            // Get the data in the li tag
            // Get all the Li's first
            // Then loop through the valid data in li
            string strHtml = HttpHelper.DownloadUrl(url);
            HtmlDocument document = new HtmlDocument();
            document.LoadHtml(strHtml);
            string liPath = "/html/body/section[1]/div/div[@class='market-bd market-bd-6 course-list course-card-list-multi-wrap js-course-list']/ul/li";
            HtmlNodeCollection liNodes = document.DocumentNode.SelectNodes(liPath);

            List<CourseEntity> courseEntities = new List<CourseEntity>();
            foreach (var node in liNodes)
            {
                CourseEntity courseEntity = GetLiData(node);
                courseEntities.Add(courseEntity);
            }
            return courseEntities;
        }

        /// <summary>
        ///When we get the data, it should be saved
        /// </summary>
        /// <param name="node"></param>
        private CourseEntity GetLiData(HtmlNode node)
        {
            CourseEntity courseEntity = new CourseEntity();
            // Start here
            HtmlDocument document = new HtmlDocument();
            document.LoadHtml(node.OuterHtml);
            string aPath = "//*/a[1]";
            HtmlNode classANode = document.DocumentNode.SelectSingleNode(aPath);
            string aHref = classANode.Attributes["href"].Value;
            courseEntity.Url = aHref;

            Console.WriteLine($" course Url:{aHref}");

            string Id = classANode.Attributes["data-id"].Value;

            Console.WriteLine($" course Id:{Id}");

            courseEntity.CourseId = long.Parse(Id);

            string imgPath = "//*/a[1]/img";
            HtmlNode imgNode = document.DocumentNode.SelectSingleNode(imgPath);
            string imgUrl = imgNode.Attributes["src"].Value;
            courseEntity.ImageUrl = imgUrl;

            Console.WriteLine($"ImageUrl:{imgUrl}");

            string namePaths = "//*/h4/a[1]";
            HtmlNode nameNode = document.DocumentNode.SelectSingleNode(namePaths);
            string name = nameNode.InnerText;

            courseEntity.Title = name;

            Console.WriteLine(Course Name:{name}");

            courseEntity.Price = new Random().Next(100.10000);  // This is an advanced content that cannot be done by ordinary means (he has his own algorithm)
            return courseEntity;

        }
        #endregion

        #regionGet Ajax request data
        /// <summary>
        ///1. Match the page and request URL
        ///2. Obtain the requested data
        ///3. Parse data: Create entity HashTable based on JSON format data
        /// 
        /// </summary>
        /// <returns></returns>
        public void GetAjaxRequest()
        {
            string url = "https://ke.qq.com/cgi-bin/get_cat_info?bkn=449651946&r=0.36532379182727115";
            var ajaxData = HttpHelper.DownloadHtml(url, Encoding.UTF8);

            Hashtable hashtable = JsonConvert.DeserializeObject<Hashtable>(ajaxData);
            string result = hashtable["result"].ToString();
            Hashtable hashResult = JsonConvert.DeserializeObject<Hashtable>(result);

            Dictionary<string.string> dicResult = JsonConvert.DeserializeObject<Dictionary<string.string>>(result);

            string catInfo = hashResult["catInfo"].ToString();
             
            //dynamic dynamicCatInfo = JsonConvert.DeserializeObject<dynamic>(catInfo);
             
            //Hashtable hashcatInfo = JsonConvert.DeserializeObject<Hashtable>(catInfo);

            //foreach (var hashItem in hashcatInfo)
            / / {
            // JsonConvert.DeserializeObject
      
       (hashItem["1001"].ToString());
      
            / /}

            //Hashtable cat1001 = JsonConvert.DeserializeObject<Hashtable>(hashcatInfo["1001"].ToString());
            // console. WriteLine($" category: {cat1001["n"]}");

            //Hashtable cat1002 = JsonConvert.DeserializeObject<Hashtable>(hashcatInfo["1002"].ToString());
            // console. WriteLine($" category: {cat1002["n"]}");

            //Hashtable cat1003 = JsonConvert.DeserializeObject<Hashtable>(hashcatInfo["1003"].ToString());
            //Console.WriteLine($" category: {cat1003["n"]}");

            //Hashtable cat1004 = JsonConvert.DeserializeObject<Hashtable>(hashcatInfo["1004"].ToString());
            // console. WriteLine($" category: {cat1004["n"]}");

            //Hashtable cat1005 = JsonConvert.DeserializeObject<Hashtable>(hashcatInfo["1005"].ToString());
            // console. WriteLine($" category: {cat1005["n"]}");

            //Hashtable cat1006 = JsonConvert.DeserializeObject<Hashtable>(hashcatInfo["1006"].ToString());
            // console. WriteLine($" category: {cat1006["n"]}");




        }
        #endregion 
    }
Copy the code