From de17741b9aa6b9ac12d42b89d03522a286eadc4a Mon Sep 17 00:00:00 2001
From: sleptworld <qwin7989@gmail.com>
Date: Mon, 21 Mar 2022 16:25:32 +0800
Subject: [PATCH] redo

---
 __pycache__/main.cpython-310-pytest-7.1.1.pyc | Bin 0 -> 751 bytes
 app/__pycache__/__init__.cpython-310.pyc      | Bin 0 -> 138 bytes
 app/__pycache__/app.cpython-310.pyc           | Bin 0 -> 1477 bytes
 app/app.py                                    |  29 +++-
 main.py                                       |  37 +++--
 spider/__pycache__/__init__.cpython-310.pyc   | Bin 0 -> 141 bytes
 spider/__pycache__/parse.cpython-310.pyc      | Bin 0 -> 2811 bytes
 spider/__pycache__/request.cpython-310.pyc    | Bin 0 -> 719 bytes
 spider/__pycache__/spider.cpython-310.pyc     | Bin 0 -> 4300 bytes
 spider/parse.py                               |  93 +++++++----
 spider/request.py                             |   9 +-
 spider/spider.py                              | 155 +++++++++++++++---
 12 files changed, 245 insertions(+), 78 deletions(-)
 create mode 100644 __pycache__/main.cpython-310-pytest-7.1.1.pyc
 create mode 100644 app/__pycache__/__init__.cpython-310.pyc
 create mode 100644 app/__pycache__/app.cpython-310.pyc
 create mode 100644 spider/__pycache__/__init__.cpython-310.pyc
 create mode 100644 spider/__pycache__/parse.cpython-310.pyc
 create mode 100644 spider/__pycache__/request.cpython-310.pyc
 create mode 100644 spider/__pycache__/spider.cpython-310.pyc

diff --git a/__pycache__/main.cpython-310-pytest-7.1.1.pyc b/__pycache__/main.cpython-310-pytest-7.1.1.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..0e45eb08ac42995d2082c4c572ec965e5f0ac722
GIT binary patch
literal 751
zcmYjP&5qMB5cW^fq;0w_d*cBXDH2K!NFX7E$_*}ipqz4vys_1~O;WGzYE{%qpyj&n
zuzNxrc?RCXS5CYF5`uBcRy)#+=g;vs-()=+4G_rFug@>OV}ySC;Gs|mrl4jCh9iz^
zRN(F;4J8PpCMZJeE2Qo<Q4w`Mt>dO&^gBPO2hFe;c0R6?BH<zLeL=-YobYHx^fQs%
zj>YI2-C^j#vrg0a7rckgQ|yLsl&WRLtX#Jc4rWSnp+7LQf+&73E;qthQ~CCL@t{M{
zAE0Im<}<cvi8;1}6Hc!Q4|sS(-vR8mXpeSij`!Ho9p=3oyeB(y8{DBo)x(~lG&*-d
zE3VAPvxT+Fyv%c>*o(|mnkkW0qP0TjOOrQDw%O7+Iv3WVlsanJ+)*tqU9=DkfEz#^
zT0n2oP<x;*kgc>j09O+qhsH8(Q(E->i>2??Ltmgwsc%S+!T)`hFV>C7t=X((uJn2-
zDr<7#xoKo|z@fG-@PHz?U{1l1PXQq~#snvSdc6TTCHj&7a9ozYuPoDvi!U~^wz4%Y
znW}Ahj2@S2Yk}#E86&h`MyAD;mR7j_G3BEB+yE~7_LAu$0^AvBH(oI^-2$`2(NR#G
z{LcYTew!T`E@EceR&ssB*s?~HC9t_^I}#q4NjJu;X3aOXcnv=?GCuVLB{+eBV;YYq
HIQaVy0cXX;

literal 0
HcmV?d00001

diff --git a/app/__pycache__/__init__.cpython-310.pyc b/app/__pycache__/__init__.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..8670a5658f504a8a497f3b2ff08c8eed81a44bd7
GIT binary patch
literal 138
zcmd1j<>g`kf;00?lR)%i5P=LBfgA@QE@lA|DGb33nv8xc8Hzx{2;!Hnenx(7s(wjv
zX?CW5K~a8IYH~@jeriExN@{LqvVLMgfqr~^W?p7Ve7s&k<t+}I-29Z%oK!oIF~v+k
Hf`tJ9l>8nF

literal 0
HcmV?d00001

diff --git a/app/__pycache__/app.cpython-310.pyc b/app/__pycache__/app.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..365ac13e5167a6bb4af676f5d751a6cbb76af2f3
GIT binary patch
literal 1477
zcmZ8gPj4JG6!*WGot?}sY3P9n)B~riK-mKl>Y=KNN`WF2kkXSSWQMU5W;6d{dx2CF
z37Vb}A7S^{&%w9wl@nh9sfcG!QcY^fZ^nMl@A>z8Kew}4ieO9+?*I5}gwWr9F&-2a
zk6@Zl!AK-=g}DEc3fF{_nsSPVeO5<2!stsRsbnvaWLJd85+9>Q^bedxM+@TS-{sn<
z$8FQ7!nWGYpEUB@lc%;;-;|9qg^wE#28%~9%^?_tIF^W$DV9@76g?rKQpA~zWc(8G
zNKRw|-=S*ic0TL6sw{F_w#|{7ceiR69@)+Cb^Lq`)BFj>q7B}n9tqMDiPlKZExqz<
z*3h2#uZ?<a<13;+kr>vK(tDF0e~!N2yQJWycDhQhh&lKgy}&))?D%}Eq^GOw0Xn}Y
z(J=}+LX`eO`mB>4ud*Kd*wSZBLe%U9UeF_+)$7WZ!gQrnn$JwxoL1_3xtP02ZZ4Xl
zZ27!VzX<iSYOJW*w&VL^#Dfrl1|3|Q08|ZaM=mWPpRsvUD95VOSjX%|r(9%nb7ovT
zRCVcbsQwH#c~)qZTP21FYV%V!eFkTqXx-`sbIhpfM0<IK5}f(KS4i)|&c6rCpW0e2
zty!Oy%TBi|5QJGOpS>=NW!`mu9(ETl5u$8LE5roOnR{T;I3X#HNlIcI<Am(uJ<R^g
zSd8ytMxNgt!9Bb=TwpkOGkmLk`$M}whM!Y(33RVQ)1iUcB{XA&da{o$DYS(2^bds5
zIke{(y~0Ne8elngrY?XRxTtF6g6Vf4)!?Dvajdke4sZu^2i$xnMBr_p?L*iyo+5_l
zIK>%(`|sbr|0wzv2oJym`OByH0nG1jAYf$4hW2Q~wrJ}WIRkk3DZJCK&;V@{^<;|y
z(k#G^K*<n^y+YAyvPrf8H~deBf3HLRE)-d$Lo;_kE81R*DFz46awCei?kZ)KbZ?1n
z1j5l=O6_mYqw}>|gWgtc+3*RdpissDxt>RdHJ`r8_ZEvT3gth7IOF9+eoH|EDR4L^
z8Q#Uu?~FMNvBT{rUQSJ3D<NDeMBU0YydmK-A<ox%HF$<+u?Kned)|EDjW07YzL(n5
z@3EUXm;@2VUK5(WPTtR^!MB*`ftum<91c$}JUWlx&93){kKTd3aBfs5l=a12@?f~_
QL$73$fJx9U&hQ@nAHXVO%m4rY

literal 0
HcmV?d00001

diff --git a/app/app.py b/app/app.py
index 8eb50f8..fb8e24d 100644
--- a/app/app.py
+++ b/app/app.py
@@ -1,5 +1,5 @@
 import asyncio
-from spider.parse import ParseConnector, FrontItem
+from spider.parse import ParseConnector, EndOfItmeLines
 import spider.spider as spider
 
 
@@ -13,7 +13,7 @@ class Application:
         self.__parser_loop = asyncio.new_event_loop()
 
         if parseConnector is None:
-            self.parseConnector = ParseConnector(loop=self.__parser_loop)
+            self.parseConnector = ParseConnector()
         else:
             self.parseConnector = parseConnector
 
@@ -22,19 +22,30 @@ class Application:
             self.tasks = {}
 
             for task in tasks:
-                assert isinstance(task, spider.SpiderTas)
-                self.tasks[task.tag] = task.get_tasks()
+                assert isinstance(task, spider.SpiderTask)
+                self.tasks[task.tag] = task.create_tasks()
 
         elif isinstance(tasks, spider.SpiderTask):
             self.type = "single_spider"
-            self.tasks = {tasks.tag: task.get_tasks()}
+            self.tasks = {tasks.tag: tasks.create_tasks()}
         else:
             raise TypeError
 
+    def __creat_tasks(self):
+        tasks = []
+        for key in self.tasks.keys():
+            item = self.tasks[key]
+
+            tasks.append(*(item))
+
+        return tasks
+
     async def start(self):
         self.parseConnector.start_parse()
-        for r in asyncio.as_completed(self.tasks):
+        for r in asyncio.as_completed(self.__creat_tasks()):
             completed = await r
-            await self.parseConnector.add(
-                FrontItem(completed.type, completed.content, completed.cb)
-            )
+            self.parseConnector.add(completed)
+
+        self.parseConnector.add(EndOfItmeLines())
+        self.parseConnector.queue.join()
+        self.parseConnector.process.join()
diff --git a/main.py b/main.py
index 82ac057..186caca 100644
--- a/main.py
+++ b/main.py
@@ -1,25 +1,32 @@
-from app import Application
+from app.app import Application
 from spider.spider import SpiderTask
 from spider.request import Requests
 import asyncio
 
 
-def main():
-    tasks = [
-        SpiderTask(
-            "news",
-            Requests(
-                "http://www.baidu.com",
-                [],
-                "get",
-                {},
-            ),
-        ),
-    ]
+def p(content):
+
+    currentConfirmedCount = content["results"][0]["currentConfirmedCount"]
+
+    print(currentConfirmedCount)
+
+
+async def main():
+
+    news_task = await SpiderTask.init(
+        tag="news",
+        req=Requests("https://lab.isaaclin.cn/nCoV/api/overall", "get"),
+        headers="",
+        dataType="json",
+        parsers=(p,),
+    )
+
+    tasks = [news_task]
 
     app = Application(tasks=tasks)
-    asyncio.run(app.start())
+
+    await app.start()
 
 
 if __name__ == "__main__":
-    main()
+    asyncio.run(main())
diff --git a/spider/__pycache__/__init__.cpython-310.pyc b/spider/__pycache__/__init__.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..438f259e95d7a6e062597e939a8c00d6b2bcdd71
GIT binary patch
literal 141
zcmd1j<>g`kg0N|(Ng(<$h(HF6K#l_t7qb9~6oz01O-8?!3`HPe1o6v2KO;XkRllUT
zG&@tjpeR2pHMyi%KeZq;B{erQS-%*>DAJFQ&&<m#iI3MSsJz8tlbfGXnv-e=GO3se
INU$&f0Kh^X)Bpeg

literal 0
HcmV?d00001

diff --git a/spider/__pycache__/parse.cpython-310.pyc b/spider/__pycache__/parse.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..387493a9d56a8352d80fd23688b029df660e7281
GIT binary patch
literal 2811
zcmZ`*O>Y}T7@nD({jg&@A#GDq5FkEl1WoAyI205V5mbcSphl`NNQ=8ONj6=t-I=wj
zTCRkYNImffI3>sarM+_E%7H@*h3B1(^Wl`S=Jh+@&phwP?4a4KGnC(Me17jz#MtlD
zI9((ftEe`IP)zZR^?8?b(nTivUe|NF2VHh$-|zaIeZ!Pg{ytOwp6CW>1u8@<L@Puq
zQZ=+{I^6TRk*ce?eb%j^zoDAwZ(>$mweGO?{2w@zwRst>^|GvYFVoP&4KqlzwQW&0
zza6AGb$>t7BVEdkbf^<ra=HjKR#ELm2+g`&v93^D2^{Jv568%|etT4GjEY-@?k616
zriFB>Yp9l(Q0xwS$~#XXN@;brWxyG8;@QV5JA=MnDeUNex-v9_U7ZwmMGsS@`)RUb
zNii$Kp0WDc@KG7Xahj(^9EY?yp_F`l;RMLF>39bKF*)z`bsU%VIPMSBC?mZQ#}7um
ztQs+mDUuURW{&y-2&(P|5o>VQbB-wTgH@V#9pZ#2?J1$9@=&GrwcO@9*b>WZD^k7+
z_GLFzAxMnM+PWF!j>B@!7QJF*<7A+8IhUYY=f&oup)SLtM%i*inDtSf6zL#Od>lG0
zq!S^I_Sd0I#Dv9c;z9C>gcK7F>5(}M@dt`zf&I+|Gz?kVyaMsmH*_nfn?=%2#*YnG
zkwolKwbUPk0S|bcj~9>cbjlo&Y0_3L67vwHm*z$3*}@n??b!W)Yb;<2nZ@~I!~_>Y
zybzxFM@aE^!*4w|G~=9g$or_+_aUa%sqyNk<P)|9msZ|>t?NBul656);+L(jb9H;`
zR?*krrMb49b2O7*c(m;jM8Wph0SC1LRLet2fjuQyl)fE-;Ob&D8zE4O<0R`@8^@K;
zId>x#uh5o6#!@)Gd=|qok!{&QY5Oy5x&zAQ(awp^vgz*5Vy|J-%3{mVs9HJ@M0%hS
zzoqAfdJ{T*TIhzf+BwwEP-<*^b-X-b2kekN76ooxh&^F0-eq?gdc8x0J>7jG4&+4a
z`iDq(cUc~B^mA4OO6;NMhf}k2f4c4=d%)XXXLCGnkq?TUG~a&fAwpxj9hTm<F3Ngp
z(;N|!C)z<RYemoAkJTX8mFtp|mj2Mdt=eAcKd7=BA}23piX1f);AKd$gk#>tI{PYw
z;8H}qE+lV63(o>k=i--(p7_(ehrx5MMeMAi+M5tl*HTUAP8>PXrSh&+f%dnA3RQ%l
z4^&OnkwHT>ryBT=6o<pL%GL(uw>3aT_8XW^GWA<*4Y!<)Yl`A3y19#PB<5*cw?$_i
zVI3lqQo1+qV6k}{qKvkXIPH#BZCQo)2Q+j%KEsalr_V5>ifqKSi|os1fgXG1x%^Ll
zv_{K`*a_Q3gphYY6P0!nBc20M`bjph+Bj4-ESdpShH<Pky!s2;l{l&M79TI21LP(d
zGU^#;CO~8|2Z}~3P!mp0S|BqP9$b0BF)0ZgRj|2<P;>lw!%<=0#6stN<|?$dsF<Yf
zX5o50rM^PH<FA3NSm*NJ@!||zOgcjyA*8!GlnM%=RBXgdv*3H|C-Im+uClnJ(v2(u
zIa<i>Y!}Ee_j=(M0Z}~>a6SGjy#rFY@i?Se2hlWC-7UQiJ_~}PXDH4j{_JMER=EZ}
zD(?P<U{%&3kS-?5K=pKgkR!=qF`~}V(VZLi3|=n0V|m`KCp&4ToIXnty?3UFG27~O
z>}lVCs3Yv)YAz5dF-ZaWB6<;pEg}CJUzs8B)D~xmJo61g>6(a~W@YZIXI|!I$TAKP
z^1_b<iQFdO<K>xSPi7%!AsyFVmX6JbSnS><^ATyEkhnp@<>V9L9sqiwK1DrB%yk|G
zEgAWZONK@|?YVM38Dtp}1Rj2St#>bR?`ApQA7w>)6z_m-oOztJobl-L6$nc3&TekU
gmw_mwo7HF1m&Bi?C$)hup9S9XTK+;Snp<4_4@?^*mH+?%

literal 0
HcmV?d00001

diff --git a/spider/__pycache__/request.cpython-310.pyc b/spider/__pycache__/request.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..d31646d7385fedaf3e95a58054486f04fb151a80
GIT binary patch
literal 719
zcmYjPJ(JTg5Y?_^#|gQ+K*jw8H!&1648z>a!O$=%fXa;~v6P&_j(n0_gmDG;XYMbl
zrJ|(efMInZLw4ppYqcLwTFacAOc>eIAJ0~|z}UCw92+O+g5c&viYaJV3njEzB19h>
z-ilHHd&3l0;)W^l70W~=?^&LHry7=H5N~VegLqX}K8KK9E}Le#YIGID$M%4n3xa!2
zq*)1yl{khms7#^eiW7tunx8Y3=tT1kXuigCw$4<lMmI^BDj?x;$lmGeUOVp=f%mov
zX{-H}Q6bq_(|1AGp5zl!IapjH;gaB<5DhRe$p$2bK`7RPoeg~d_4ow_P7yu`Vv3R|
zEb@dJ8{_SW>cjZMuG6orHP&Y2+c8nhaJpIBH15Q-|FK(UPiW(3QCyi;7vA-s>Y}se
zqprLwbXP0g)>Yx+!WMRazj?O{nUwXm_EN^CZc3DnQkbED6esZc;b9c>Ltc)HMLvpk
z+hwbz3==6^qi7_wo=SP$FPnc6?!2{eE>=W3ixf|JM5ob><BM>Qi86)>g1AG(KgjWS
zD)84dIla5jTI6X+{jRIGn|<T>!opc)8u}x?Hd{AeuByFU?0BSfFC67U_F{kDm-J+r
Mi}#+fDLmqT0VL#{@c;k-

literal 0
HcmV?d00001

diff --git a/spider/__pycache__/spider.cpython-310.pyc b/spider/__pycache__/spider.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..0283d2f0229331845d4e233ce3c226f94885ffd1
GIT binary patch
literal 4300
zcmZ`+&2JmW6`z^?;F4TXvc$-i?Id>6reM)ZE}GOy>$pzkkEl)6G*VEMNI}q?l|*XE
zrDm2$EKq<Q8EASi3N)9ZC<+Pa#s5To$fbY7Tzm4Nz4THzj@{pz)k>sn?_%E0_q;dn
z_dYh6m?$wke}41rn`WA^e^F!fanZPfqCW?bO!Ao3voEi6DWnsNu2Xk7L*0$tu2=U^
zr+I$e$JdJs-Jl+Hi}j-I_v2ExTrb;tAr89}^$E^yFd4|=T_%fBu<O(-vLwrQS$z`y
zp`1Yf#IC4MNxs5rm51PiEkER-HEx2BqOC!sQ;mA@x=MQKQW|w@!noIZgRXI#gEYaA
zw>s#@5zSw0#&Ppz9JOeh(T6s?f}+0#5-|wJ>LPM>S>3hNv(&e=U|A7nYNg+nky>r)
z&6Tv74)k&@FhQfyYj&eX!;~70ZXyRU)x$<(YtW3d5#{3y<1Q^Pt(ww@AN^|a+RDn(
z@(oj5`FN4)*RC4>`g=<sT)TRosS}uM$)ov&jiehbq<XN~Ug)c&6SY#k5D^dEc56Xf
zW)42{{adPtm1Pu7Dh`;>=`C^F-y(VQhw1`l5k-d}DI2m~PD)x~_xUp2U|+R+DJq2)
zE;;I@M|>&Z2GXQZ1JyL`6mm6-&oj47>|8<7{{YEvS@Ot{B66h@c^zN6Q6UN<XC1w3
zVvfxLM5vXdFAI0wdNC?R<uy)#k;OIla4pU8EpWVDUJo<i)F)&`P6A35c}|{)Cgn6{
zxpGEU?|J~#OVNonA!p@D>^n^ypmj>lp*3S?3VB+dL8~gy%9pXrOY#-@D!#L_2q>O6
zr6c_tCvC19M@3tvpd+o@NzWAeO{F8HO(2_TbM;m~YLVjehw6v7ehG~sAF>7;3Q#_D
zK!t=J8qPXVcj$t8soyE!8%Ua49D1N7(o#(<D{B*FbqZUlIU=V)%yi?3%SIywR<$`f
z*7qswX1x)&b!w_(0|)$Dz^Dn@%-0s@CRl6t+Il0B>MXizzHxOFuN^yDogpS@!f~*`
zMzB6@G)`hr6Keb*gD26z!gPD)i0}D)KEdp>bh#B^N&XPb3n*Wq=yM=<c*;7sLCNp&
zogfvsS1}B5f1VWgJlvH-h%RhIO&G7Pi6cI1jwv+T$wr#?&BS8d2F_P9Wh&ydR#dOy
zGa}}<sd*@79mbJtwj)jEl6-c{D&V;6E=gX)IxDxh9hL<@di}4rCr9L_xnrj4VIc|c
z&<1y*aS379#P1nQ0^ay;xWm7I@RALfda%xR1X`lwpv~24>Y@enknW~X7qgM8?03~2
zXDIgkAx{gPV6SNVOG9U0?9yJI^yb*U;B4#s*8Q}ct)l*2fweQAh#hCgebM@%YghiX
z_px0UT0XuU<6}T0d!ZdK?Q`r|%J`u(?AdTcTkYqZUCqvZW_{s69(SKs{$XpUH#>Tw
z{_D@#=YRxBoSmAVv@2-E5O>z#V=EryE&JuWd{gKLv<p^JRPvXN+i#{DEwbnQq0mX#
z{1Qd~2jmV-XM5S9u<w2GCR>7!eV_g1lN~V>9eA@%M_sV&zzay{p2)^IOMN>|p_T2r
zzk%bB%*DIw<TF;-7i`blKiH9@)hSp-Uxlba-oNAYs&>Q!1b&T?;>bt=?b1;@_2Fei
zrwm`jUv%aV4$jm4PjPkh{uaf0B!BtgKOyxhqOrf$Y{zgG-$h$dfHiJMC%syjxu^o%
zCPi`UwT$15(v3u_b2P6=4yz%PUZm(gGjBpWsPEITt6;<?Jcvn+6h5}*+GC8Ld9Qk)
zBHd4VIx@U<pJxVaW2y~XLTz2fM*0GXxP2Dku?lN+bBlaD`%R@(5>x1(;uV1>AA5Y^
z-`lT`SZWKLq*3Ny=P$)cs~PLd^J+^GBr?}LVI0V)UZxK6&&FF<$)InXIO?fasl#vf
z`%zCCf1u*HeG_hkB8VxV*-6?xh?^v-akkt2x~q^_)VxfdAJJkb?e+~<>W8$ymGCX9
zSuw*=_TELez5+7MOJ`gjiV6=QR+Sv@6Y=zvPjzAIp;tkDnxogLVwQL+o=$tGxi8*;
zlioi0f(saPS$TG0{WE0+cM!YC4sxVrcu_MLf-|;3%WK66LZ=nrFaffns6|ZMM$v=^
zal6X5N7Ar)yY9OB9>y#TTI4A6hw4<mLeaNDzzJ8CA>!<S>!NiFY!42=x&c@h94WsD
zTSGiX9g&1}95R0O){yPFNBr9na;$>vH;>U<@1RzsXx(#BD^V>iV~-%utq_Su@DZ9y
zQ_4mbj@!s%)p_u!-XUT^vq-g9h>%}VQ$)@YnIuvnQU$3w87dAh;aCV#1oj;iO{s#o
z4F~|=6+Q(ZR78oFL?9|+yE>xWBP)ClOUB$B;hQd%nENlWlqq+}=@Q8i<ccK`jMGp0
zP_hl9wAC@zOj$;8bYMq+q#bGdDZV{B!pz8&F;0jJN-&8N{Q(HlSl)rRwr=$`sw2A0
z!PQd4aZ-0LOQ(1zNQ<4)UKuk&C|@8=00eCGu*sEmXHXR(xyI`b%&|6+>E&&7Wy<+6
zYRDe+tzN60nDU6hsW)iSJdp(=KOsU1pCU_C<bQIN*|hC>_Yx2?NDF_$Y^%2lbmqfs
z>8)@VhU-6dpSXN1Z^~DV-P5SX)HPz^!1OG%NRC`X)?Xj9zpEJk8!dVtWP5HzFXP~S
z#s|>tS&&fTOCZUi!81%h$Jk9}JxWImVjn9lBH(*QPadE3!jlz&A`=#WG&U7}o{KR>
z+RqOEofa&Ej9J6sa7|oauANl$f29a{DkO4|2!SzE{CB8E396#RO}!2Rc*ISu4+0(e
zL`4~)`WX?rBjesolGqxmaq#1%ZqUdth|vA3UlRF{$dPtzjl8|hRcTXw1teg|E`@j^
z9Py3oipNu)_*(d-M`GIbv%J&&Up_AU@KG=myjyiD(-o%@22SW?nePd;L^~fL*~(`@
zOn%s6+PD#Zx)r6U*&nWtc--!-Tj9-QD~a*no#M|(&o^(jvfHqZ%w7N`*w2&31edbE
Z=*t8kO>+Ylz-5JqS5x*o<phOk_x~wUjT!&|

literal 0
HcmV?d00001

diff --git a/spider/parse.py b/spider/parse.py
index df3b515..93562e9 100644
--- a/spider/parse.py
+++ b/spider/parse.py
@@ -1,6 +1,6 @@
-import threading
-import asyncio
 from collections.abc import Callable
+from multiprocessing import Process
+from multiprocessing import JoinableQueue, Pipe
 
 
 class OutPutItem:
@@ -11,6 +11,7 @@ class OutPutItem:
 class FrontItem:
     def __init__(
         self,
+        status_code: int,
         contentType: str,
         content: str,
         parserFunction: Callable[[str], OutPutItem],
@@ -18,38 +19,74 @@ class FrontItem:
         self.type = contentType
         self.content = content
         self.parserFunction = parserFunction
+        self.status_code = status_code
+
+
+class EndOfItmeLines(FrontItem):
+    def __init__(
+        self,
+        status_code=None,
+        contentType=None,
+        content=None,
+        parserFunction=None,
+    ):
+        super().__init__(status_code, contentType, content, parserFunction)
+
+
+def parsing(queue, conn):
+    while True:
+
+        try:
+            item: FrontItem = queue.get()
+
+            if isinstance(item, EndOfItmeLines):
+                break
+
+            item.parserFunction(item.content)
+
+        except:
+            print("something wrong")
+
+        finally:
+            queue.task_done()
 
 
 class ParseConnector:
-    def __init__(self, loop, tr=None):
-        self.__queue = asyncio.Queue()
-        self.__loop = loop
-        if tr is None:
-            self.__threading = threading.Thread(
-                target=__NewTr,
-                args=(loop,),
-            )
-            self.__threading.daemon = True
+    def __init__(self, process=None, conn=None, queue=None):
+        self.process = process
+        self.conn = conn
+        self.queue = queue
+        self.finished = False
 
-        else:
-            self.__threading = tr
+    def __del__(self):
+        self.conn.close()
+        self.process.close()
 
-        self.__threading.start()
-
-    async def add(self, item: FrontItem):
-        await self.__queue.put(item)
-
-    async def parsing(self):
-        while True:
-            item: FrontItem = await self.__queue.get()
-            item.parserFunction(item.content)
-            self.__queue.task_done()
-        await self.__queue.join()
+    def add(self, item: FrontItem):
+        self.queue.put(item)
+        if isinstance(item, EndOfItmeLines):
+            self.finished = True
 
     def start_parse(self):
-        asyncio.run_coroutine_threadsafe(self.parsing(), self.__loop)
 
+        if self.conn is None:
+            parent_conn, child_conn = Pipe()
+            self.conn = parent_conn
 
-def __NewTr(loop: asyncio.AbstractEventLoop):
-    asyncio.set_event_loop(loop)
-    loop.run_forever()
+        if self.queue is None:
+            self.queue = JoinableQueue()
+
+        if self.process is None:
+            self.process = Process(
+                target=parsing,
+                args=(
+                    self.queue,
+                    child_conn,
+                ),
+            )
+            self.process.daemon = True
+
+            self.process.start()
+
+    def is_parsing(self):
+        return self.finished
diff --git a/spider/request.py b/spider/request.py
index 5ba53a3..e605578 100644
--- a/spider/request.py
+++ b/spider/request.py
@@ -1,13 +1,20 @@
 from typing import List, Dict
+from collections.abc import Callable
 
 
 class Requests:
     def __init__(
         self,
         url: str,
-        group: List[str],
         method: str,
+        group: List[str] = None,
+        rule: Callable[[str], str] = None,
     ):
         self.root = url
         self.method = method
+
+        if rule is not None and group is not None:
+            raise TypeError
+
         self.group = group
+        self.rule = rule
diff --git a/spider/spider.py b/spider/spider.py
index 6330f19..584aa1d 100644
--- a/spider/spider.py
+++ b/spider/spider.py
@@ -1,8 +1,22 @@
 import asyncio
 import aiohttp
-from request import Requests
+from .request import Requests
+from .parse import FrontItem
 import urllib
-from typing import List
+from enum import Enum, auto
+from typing import Tuple
+from collections.abc import Callable
+
+
+class SpiderTaskStatus(Enum):
+    INIT = auto()
+    PROCESSING = auto()
+    SUCCESSED = auto()
+    FAILED = auto()
+
+
+def test(content):
+    print(content)
 
 
 class SpiderTask:
@@ -10,36 +24,121 @@ class SpiderTask:
         self,
         tag: str,
         req: Requests,
-        headers,
-        cookies=None,
-        proxy=None,
-        parser=None,
+        session: aiohttp.ClientSession,
+        parsers: Tuple[Callable[[str], str]],
+        dataType: str,
     ):
         self.tag = tag
         self.req = req
-        self.session = aiohttp.ClientSession(
-            headers=headers,
-            cookies=cookies,
+        self.session = session
+        self.__tasks = None
+        self.__wrong_list = []
+        self.__parsers = parsers
+        self.status = SpiderTaskStatus.INIT
+        self.finished = 0
+        self.dataType = dataType
+
+    @classmethod
+    async def init(
+        self,
+        tag: str,
+        req: Requests,
+        headers,
+        parsers=(test,),
+        dataType: str = "text",
+        cookies=None,
+        proxy=None,
+    ):
+        session = aiohttp.ClientSession(headers=headers)
+        return SpiderTask(
+            tag,
+            req,
+            session,
+            parsers=parsers,
+            dataType=dataType,
         )
 
-        self.proxy = proxy
-
-        tmp = []
-        if len(req.group) != 0:
-            for r in req.group:
-                tmp.append(
-                    asyncio.create_task(
-                        aiohttp.request(
-                            req.method,
-                            url=urllib.parse.urljoin(req.root, r),
-                        )
+    def create_tasks(self):
+        async def rq(path: str, parser):
+            async with self.session.request(self.req.method, path) as response:
+                if response.status != 200:
+                    await self.task_done(failed=True, path=path)
+                    return FrontItem(
+                        response.status,
+                        None,
+                        None,
+                        None,
                     )
-                )
+                else:
+                    await self.task_done()
 
-        self.__tasks: List[asyncio.Task] = tmp
-        if isinstance(parser, list) and len(parser) != len(req.group):
-            raise TypeError()
-        self.parser = parser
+                    if self.dataType == "text":
+                        c = await response.text()
+                    elif self.dataType == "json":
+                        c = await response.json()
+                    else:
+                        c = await response.read()
+
+                    return FrontItem(
+                        200,
+                        response.content_type,
+                        c,
+                        parser,
+                    )
+
+        if self.__tasks is None:
+            tmp = []
+
+            if self.req.group is not None:
+                if len(self.__parsers) == 1:
+                    for r in self.req.group:
+                        tmp.append(
+                            rq(urllib.parse.urljoin(self.req.root, r)),
+                            self.__parsers[0],
+                        )
+                else:
+                    for r, parser in zip(self.req.group, self.__parsers):
+                        tmp.append(
+                            rq(urllib.parse.urljoin(self.req.root, r)),
+                            parser,
+                        )
+            elif self.req.rule is not None:
+                if len(self.__parsers) == 1:
+                    for r in self.req.rule(self.req.root):
+                        tmp.append(self.__parsers[0])
+                else:
+                    for r, parser in zip(self.req.group, self.__parsers):
+                        for r in self.req.rule(self.req.root):
+                            tmp.append(parser)
+
+            else:
+                assert len(self.__parsers) == 1
+                tmp.append(rq(self.req.root, self.__parsers[0]))
+
+            self.__tasks = tmp
+            assert len(self.__parsers) == 1 or len(self.__parsers) == len(self.__tasks)
+        return self.__tasks
+
+    async def close(self):
+        await self.session.close()
+
+    async def task_done(self, failed: bool = False, path: str = None):
+        if failed:
+            assert path is not None
+            self.add_wrong_item(path)
+        self.finished += 1
+
+        if self.finished == len(self.__tasks):
+            await self.close()
+
+            if len(self.__wrong_list) != 0:
+                self.status = SpiderTaskStatus.FAILED
+            else:
+                self.status = SpiderTaskStatus.SUCCESSED
+        else:
+            self.status = SpiderTaskStatus.PROCESSING
+
+        print(self.status)
 
     def delete_task(self, r: int):
         self.__tasks.pop(r)
@@ -56,5 +155,11 @@ class SpiderTask:
                     )
                 )
 
+    def add_wrong_item(self, path: str):
+        self.__wrong_list.append(path)
+
+    def get_wrong_item(self):
+        return self.__wrong_list
+
     def get_tasks(self):
         return self.tasks