Changeset 867

Timestamp:

10/30/07 09:35:46 (6 years ago)

Author:

mattn

Message:

lang/lua/LuaScraper/luascraper.lua: modified that resolving absolute url path.

Files:

• lang/lua/LuaScraper/luascraper.lua (modified) (6 diffs)

lang/lua/LuaScraper/luascraper.lua

@@ -1 +1 @@
 local http = require("socket.http")
+local url = require("socket.url")
 local xml = require("xml")
 
@@ -7 +8 @@
 end
 -- not supported
-function result(p)
-  return p
+function result(self)
+  self.name = "result"
+  return self
 end
 -- return scraper structure
@@ -14 +16 @@
   self.name = "scraper"
   -- scrape method
-  function self.scrape(url, ctx)
+  function self.scrape(uri, ctx)
     -- create http session and parse HTML
     if ctx == nil then
@@ -20 +22 @@
       local b, c = http.request {
         method = "GET",
-        url = url,
+        url = uri,
         sink = ltn12.sink.table(chunk)
       }
@@ -39 +41 @@
           for k1,v1 in pairs(ctx.doc:select(v.process.xpath)) do
             local newctx = {top=ctx.top, doc=v1}
-            self.res[#(self.res)+1] = v.process.scraper.scrape(url, newctx)
+            self.res[#(self.res)+1] = v.process.scraper.scrape(uri, newctx)
           end
         else
@@ -49 +51 @@
             elseif string.sub(attr, 1, 1) == "@" then
               attr = string.sub(attr, 2)
-              self.res[v.process.name] = node[1]:attribute(attr)
+              val = node[1]:attribute(attr)
+                          nname = node[1]:name()
+              if (nname == "img" and attr == "src") or (nname == "a" and attr == "href") then
+                  if string.sub(uri, -1) == "/" then
+                      val = url.absolute(uri, val)
+                  else
+                      val = url.absolute(uri + "/", val)
+                  end
+              end
+              self.res[v.process.name] = val
             end
           end