Changeset 17940

Timestamp:

08/21/08 08:55:37 (5 months ago)

Author:

showyou

Message:

crochet データ取得にWebスクレイピング実装

Location:

lang/python/crochet

Files:

• chat_allsrc.py (modified) (10 diffs)
• twitter3.py (modified) (2 diffs)

lang/python/crochet/chat_allsrc.py

@@ -64 +64 @@
         def run(self):
                 #ここに通信処理を書く
-                a = self.tw.get("")
+                #a = self.tw.get("")
+                a = self.tw.getWithScraping("")
                 self.func(a)
 
+"""
+カスタムページ(自分でフィルタリングする)
+"""
+class CustomPage(wx.NotebookPage):
+
+        dataList = []
+        def __init__(self, parent):
+                
+                self.owner = parent
+                wx.NotebookPage.__init__(self,parent.getNotebook(),-1)
+                parent.getNotebook().AddPage(self,'DM')
+                list = self.list = wx.ListCtrl(self,-1,style=wx.LC_REPORT|wx.LC_HRULES)
+                list.Bind(wx.EVT_KEY_DOWN, self.myKeyHandler)
+                list.Bind(wx.EVT_LIST_ITEM_ACTIVATED,self.OnDoubleClick)
+
+                list.InsertColumn(0," ",1,20)
+                list.InsertColumn(1,u"ユーザ")
+                list.InsertColumn(2,u"発言",0,200)
+                list.Bind(wx.EVT_LIST_ITEM_SELECTED,self.OnTwitListSelect)
+        
+        def Reflesh(self):
+                t = TwDMHttpThread(self.owner.tw,self.RefleshList)
+                t.run()
 """
 最近のfriendsの発言一覧を表示するページ
@@ -193 +217 @@
 
                                 dataList.append(dataListElement)
+                
+                # Listへの追加
                 i = 0
+                import re
+                user = self.owner.tw.user['user']
                 for b in dataList:
                         j = 0
@@ -200 +228 @@
                                 list.SetStringItem(i,j,b[j])
                         
+                        if re.match(user,b[1]):
+                                self.list.SetItemTextColour(i,wx.Color(0,0,255))
+
+                        if re.search(user,b[2]):
+                                self.list.SetItemTextColour(i,wx.Color(255,0,0))
                         #import time
                         #time.sleep(1)          
@@ -251 +284 @@
         
         def Reflesh(self):
-                t = TwDMHttpThread(self.owner.tw,self.RefleshList)
-                t.run()
+                #t = TwDMHttpThread(self.owner.tw,self.RefleshList)
+                #t.run()
+                pass
         
 class MainFrame(wx.Frame):
@@ -287 +321 @@
                 notebook = self.notebook = wx.Notebook(self,-1)
 
-                # Recent
                 self.recentPage = RecentPage(self)
-                
-                # Reply
                 self.replyPage = ReplyPage(self)
-
-                # Direct
                 self.directPage = DMPage(self) 
 
@@ -304 +333 @@
                 userIcon = self.userIcon = wx.StaticBitmap(self,-1,wx.NullBitmap,(0,0),(64,64))
                 userName = self.userName = wx.StaticText(self,-1,"test")
-                twitTime = self.twitTime = wx.StaticText(self,-1,"")
+                twitTime = self.twitTime = wx.StaticText(self,-1,"---")
                 
                 messageSizer3 = wx.BoxSizer(wx.HORIZONTAL)
-                messageSizer3.Add(userName,0)
-                messageSizer3.Add(twitTime,0)
+                messageSizer3.Add(userName,1,wx.EXPAND)
+                messageSizer3.Add(twitTime,1,wx.EXPAND)
 
                 messageSizer2 = wx.BoxSizer(wx.VERTICAL)
@@ -341 +370 @@
                 self.timer2 = wx.Timer(self,self.TIMER_ID2)
                 wx.EVT_TIMER(self,self.TIMER_ID2,self.OnReplyUpdate)
-                self.timer2.Start(90000)
+                self.timer2.Start(300000)
                 
                 self.timer3 = wx.Timer(self,self.TIMER_ID3)
                 wx.EVT_TIMER(self,self.TIMER_ID3,self.OnDMUpdate)
-                self.timer3.Start(90000)
+                self.timer3.Start(600000)
                 self.RefleshTw()
                 self.SetNowTime2StatusBar()
@@ -353 +382 @@
                 # コンボボックスの中身を空にする
                 combo = self.text 
-                #name = XRCCTRL(self,'ID_NAME')
                 self.tw.put(combo.GetValue())   
                 combo.SetValue("")
@@ -361 +389 @@
         def RefleshTw(self):
                 self.recentPage.Reflesh()       
-                #self.replyPage.Reflesh()
-                #self.directPage.Reflesh()
         
         def OnUpdate(self, event):
@@ -416 +442 @@
                 if self.imageList.has_key(unicodeUrl):
 
-                        self.userIcon.SetBitmap(self.imageList[unicodeUrl].ConvertToBitmap())
+                        #self.userIcon.SetBitmap(self.imageList[unicodeUrl].ConvertToBitmap())
                         pass
                 else:

lang/python/crochet/twitter3.py

@@ -163 +163 @@
                 In: なし(クラスメンバ変数でuser,pass)
         """
-        """def signIn(self):
+        def singIn(self,s):
                 import cookielib
+                #opener.open("http://twitter.com/sessions","username=&s&password%s" % (self.user['user'],self.user["pass"]))
+                postdata = {}
+                postdata['session[username_or_email]'] = self.user['user']
+                postdata['session[password]'] = self.user['pass']
+                #postdata['source'] = s
+                en_post_data = urllib.urlencode(postdata)
+
                 cj = cookielib.CookieJar()
                 opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj))
+
+                # ログイン／cookie取得
+                r = opener.open("http://twitter.com/sessions",en_post_data)
+                print r.read()
                 return opener
-        def putHtml(self,s):
-                import cookielib
-                
-                #cookie管理オブジェクト作る
-                try:
-                        opener = signIn()
-
-                except:
-                        pass
-
-                opener.open("http://twitter.com/sessions","username=&s&password%s" % (self.user['user'],self.user["pass"]))
-                postdata = {}
-                postdata['status'] = s.encode('utf-8')
-                #postdata['source'] = s
-                param = urllib.urlencode(postdata)
-                data = urllib2.urlopen("http://"+self.url+"/statuses/update",param)
-                print data.read()"""
-        def getUserPageWithScraping(self,user,num):
-                self.setAuthHandler()
-                s = "http://"+self.url+"/"+user+"?page="+str(num)
-                print "url+" +s
-                data = urllib2.urlopen(s)
-                urlstring = data.read()
+        def getWithScraping(self,user,num=1):
+
+                opener = self.singIn("")
+                #ログイン必要？
+                s = "http://"+self.url+"/home?page="+str(num)
+                print "url+" +s
+                data = opener.open(s)
+                urlstring = data.read()
+                print urlstring
                 if num == 1:
                         return self.scrapeTwit(urlstring,True)
                 else:
                         return self.scrapeTwit(urlstring,False)
+        
+        def getUserPageWithScraping(self,user,num):
+
+                self.setAuthHandler()
+                s = "http://"+self.url+"/"+user+"?page="+str(num)
+                print "url+" +s
+                data = urllib2.urlopen(s)
+                urlstring = data.read()
+                if num == 1:
+                        return self.scrapeTwitUserPage(urlstring,True)
+                else:
+                        return self.scrapeTwitUserPage(urlstring,False)
                 
         # HTMLをスクレイプします。
-        def scrapeTwit(self,a,isFirst):
+        def scrapeTwitUserPage(self,a,isFirst):
                 import re
                 # <span class="entry-title entry-content">~</span>(最短)
@@ -215 +224 @@
 
                 return retList
+
+
+        def scrapeTwit(self,str,flag):
+                import re
+                regTwit = re.compile("<tr class=\"hentry hentry_hover\"([\w\W]*?)</tr>",re.MULTILINE)
+                regImage = re.compile("<img [\w\W]*? src=\"([\w\W]*?)\"",re.MULTILINE)
+                regUserAndMessage = re.compile("<td class=\"content\">([\w\W]*?)</td>")
+                regUser = re.compile("<a href=.*?>(.*?)</a>")
+                regMessage = re.compile("<span class=\"entry-content\">([\w\W]*?)</span>")
+                regTime = re.compile("<abbr class=\"[\w\W]*?\" title=\"([\w\W]*?)\">")
+                str = str.replace("\t","")
+                str = str.replace("\n","")
+                a = regTwit.findall(str)
+                #print a
+                resultList = []
+                for aa in a:
+                        resultData = {}
+
+                        b = regImage.search(aa)
+                        resultData['image'] = b.group(1)
+                        c = regUserAndMessage.search(aa).group(1)
+                        d1 = regUser.search(c)
+                        d2 = regMessage.search(c)       
+                        resultData['user'] = unicode(d1.group(1),'utf-8')
+                        resultData['message'] = unicode(d2.group(1),'utf-8')
+                        tm = regTime.search(aa)
+                        resultData['time'] = tm.group(1)
+                        result = []
+                        result.append(resultData['user'])
+                        result.append(resultData['message'])
+                        result.append(resultData['time'])
+                        result.append(resultData['image'])
+                        resultList.append(result)
+
+                return resultList