Changeset 17940 for lang/python/crochet/twitter3.py

Timestamp:

08/21/08 08:55:37 (5 months ago)

Author:

showyou

Message:

crochet データ取得にWebスクレイピング実装

Files:

• lang/python/crochet/twitter3.py (modified) (2 diffs)

lang/python/crochet/twitter3.py

@@ -163 +163 @@
                 In: なし(クラスメンバ変数でuser,pass)
         """
-        """def signIn(self):
+        def singIn(self,s):
                 import cookielib
+                #opener.open("http://twitter.com/sessions","username=&s&password%s" % (self.user['user'],self.user["pass"]))
+                postdata = {}
+                postdata['session[username_or_email]'] = self.user['user']
+                postdata['session[password]'] = self.user['pass']
+                #postdata['source'] = s
+                en_post_data = urllib.urlencode(postdata)
+
                 cj = cookielib.CookieJar()
                 opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj))
+
+                # ログイン／cookie取得
+                r = opener.open("http://twitter.com/sessions",en_post_data)
+                print r.read()
                 return opener
-        def putHtml(self,s):
-                import cookielib
-                
-                #cookie管理オブジェクト作る
-                try:
-                        opener = signIn()
-
-                except:
-                        pass
-
-                opener.open("http://twitter.com/sessions","username=&s&password%s" % (self.user['user'],self.user["pass"]))
-                postdata = {}
-                postdata['status'] = s.encode('utf-8')
-                #postdata['source'] = s
-                param = urllib.urlencode(postdata)
-                data = urllib2.urlopen("http://"+self.url+"/statuses/update",param)
-                print data.read()"""
-        def getUserPageWithScraping(self,user,num):
-                self.setAuthHandler()
-                s = "http://"+self.url+"/"+user+"?page="+str(num)
-                print "url+" +s
-                data = urllib2.urlopen(s)
-                urlstring = data.read()
+        def getWithScraping(self,user,num=1):
+
+                opener = self.singIn("")
+                #ログイン必要？
+                s = "http://"+self.url+"/home?page="+str(num)
+                print "url+" +s
+                data = opener.open(s)
+                urlstring = data.read()
+                print urlstring
                 if num == 1:
                         return self.scrapeTwit(urlstring,True)
                 else:
                         return self.scrapeTwit(urlstring,False)
+        
+        def getUserPageWithScraping(self,user,num):
+
+                self.setAuthHandler()
+                s = "http://"+self.url+"/"+user+"?page="+str(num)
+                print "url+" +s
+                data = urllib2.urlopen(s)
+                urlstring = data.read()
+                if num == 1:
+                        return self.scrapeTwitUserPage(urlstring,True)
+                else:
+                        return self.scrapeTwitUserPage(urlstring,False)
                 
         # HTMLをスクレイプします。
-        def scrapeTwit(self,a,isFirst):
+        def scrapeTwitUserPage(self,a,isFirst):
                 import re
                 # <span class="entry-title entry-content">~</span>(最短)
@@ -215 +224 @@
 
                 return retList
+
+
+        def scrapeTwit(self,str,flag):
+                import re
+                regTwit = re.compile("<tr class=\"hentry hentry_hover\"([\w\W]*?)</tr>",re.MULTILINE)
+                regImage = re.compile("<img [\w\W]*? src=\"([\w\W]*?)\"",re.MULTILINE)
+                regUserAndMessage = re.compile("<td class=\"content\">([\w\W]*?)</td>")
+                regUser = re.compile("<a href=.*?>(.*?)</a>")
+                regMessage = re.compile("<span class=\"entry-content\">([\w\W]*?)</span>")
+                regTime = re.compile("<abbr class=\"[\w\W]*?\" title=\"([\w\W]*?)\">")
+                str = str.replace("\t","")
+                str = str.replace("\n","")
+                a = regTwit.findall(str)
+                #print a
+                resultList = []
+                for aa in a:
+                        resultData = {}
+
+                        b = regImage.search(aa)
+                        resultData['image'] = b.group(1)
+                        c = regUserAndMessage.search(aa).group(1)
+                        d1 = regUser.search(c)
+                        d2 = regMessage.search(c)       
+                        resultData['user'] = unicode(d1.group(1),'utf-8')
+                        resultData['message'] = unicode(d2.group(1),'utf-8')
+                        tm = regTime.search(aa)
+                        resultData['time'] = tm.group(1)
+                        result = []
+                        result.append(resultData['user'])
+                        result.append(resultData['message'])
+                        result.append(resultData['time'])
+                        result.append(resultData['image'])
+                        resultList.append(result)
+
+                return resultList