root/lang/csharp/Scraper/Misuzilla.Text.Scraper/Scraper.cs @ 3473

using System;
using System.Collections.Generic;
using System.Text;
using System.IO;
using System.Xml;
using System.Net;

using Sgml;
using Misuzilla.Web.CSSSelectorToXPath;

namespace Misuzilla.Text.Scraper
{
    public class Scraper
    {
        public static ScrapeDefine Define()
        {
            return new ScrapeDefine();
        }

        private Dictionary<String, Object> _values;
        private String[] _keys;
        private ScrapeDefine _scrapeDef;
        private ScrapeDefine.ProcessCommand[] _commands;

        public Scraper(ScrapeDefine scrapeDef, Dictionary<String, Object> values, String[] keys, ScrapeDefine.ProcessCommand[] commands)
        {
            _scrapeDef = scrapeDef;
            _values = values;
            _keys = keys;
            _commands = commands;
        }

        public Dictionary<String, Object> Scrape(Uri uri)
        {
            using (WebClient wClient = new WebClient())
            {
                return Scrape(wClient.DownloadString(uri));
            }
        }
        public Dictionary<String, Object> Scrape(String contentBody)
        {
            return Scrape(new StringReader(contentBody));
        }
        public Dictionary<String, Object> Scrape(TextReader textReader)
        {
            _values = new Dictionary<string, object>();

            XmlDocument xDoc = new XmlDocument();
            SgmlReader reader = new SgmlReader();
            reader.InputStream = textReader;
            reader.StripDocType = true;
            reader.DocType = "HTML";
            xDoc.Load(reader);

            foreach (ScrapeDefine.ProcessCommand cmd in _commands)
            {
                XmlNodeList nodes = xDoc.SelectNodes(cmd.XPath);
                if (nodes.Count == 0)
                    continue;

                foreach (ScrapeArg args in cmd.Args)
                {
                    if (args.ValueConverter != null)
                    {
                        String result = args.ValueConverter(nodes);
                        _values[args.StoreName] = result;
                    }
                    else
                    {
                        if (nodes.Count == 1)
                        {
                            if (args.Value.StartsWith("@"))
                            {
                                _values[args.StoreName] = nodes[0].Attributes[args.Value.Substring(1)].Value;
                            }
                            else if (String.Compare(args.Value, "TEXT", true) == 0)
                            {
                                _values[args.StoreName] = nodes[0].InnerText;
                            }
                        }
                    }
                }
            }

            //
            Dictionary<String, Object> retValues = new Dictionary<string, object>();
            foreach (String key in _keys)
            {
                retValues[key] = _values.ContainsKey(key) ? _values[key] : null;
            }
            return retValues;
        }
    }

    public class ScrapeArg
    {
        public ScrapeArg()
        {
        }

        public ScrapeArg(String storeName, String value)
        {
            StoreName = storeName;
            Value = value;
        }
        public ScrapeArg(String storeName, Converter<XmlNodeList, String> valueConverter)
        {
            StoreName = storeName;
            ValueConverter = valueConverter;
        }

        public String StoreName { get; set; }
        public Converter<XmlNodeList, String> ValueConverter { get; set; }
        public String Value { get; set; }
    }

    public class ScrapeDefine
    {
        private Queue<ProcessCommand> _commands = new Queue<ProcessCommand>();
        private Dictionary<String, Object> _values = new Dictionary<string, object>();
        private String[] _returnKeys;
        private Scraper _scraper;

        public ScrapeDefine()
        {
        }

        public ScrapeDefine Process(String xpathOrSelctor, params ScrapeArg[] args)
        {
            String xpath;
            if (!xpathOrSelctor.StartsWith("/"))
            {
                xpath = SelectorToXPathConverter.SelectorToXPath(xpathOrSelctor);
            }
            else
            {
                xpath = xpathOrSelctor;
            }
            _commands.Enqueue(new ProcessCommand(xpath, args));
            return this;
        }

        public Scraper Result(params String[] keys)
        {
            _returnKeys = keys;
            return new Scraper(this, _values, keys, _commands.ToArray());
        }

        public class ProcessCommand
        {
            public String XPath;
            public ScrapeArg[] Args;

            public ProcessCommand(String xpath, params ScrapeArg[] args)
            {
                XPath = xpath;
                Args = args;
            }
        }
    }
}