41 lines
1.1 KiB
Python
41 lines
1.1 KiB
Python
import asyncio
|
|
from spider.parse import ParseConnector, FrontItem
|
|
import spider.spider as spider
|
|
|
|
|
|
class Application:
|
|
def __init__(
|
|
self,
|
|
tasks,
|
|
parseConnector: ParseConnector = None,
|
|
):
|
|
|
|
self.__parser_loop = asyncio.new_event_loop()
|
|
|
|
if parseConnector is None:
|
|
self.parseConnector = ParseConnector(loop=self.__parser_loop)
|
|
else:
|
|
self.parseConnector = parseConnector
|
|
|
|
if isinstance(tasks, list):
|
|
self.type = "multi_spider"
|
|
self.tasks = {}
|
|
|
|
for task in tasks:
|
|
assert isinstance(task, spider.SpiderTas)
|
|
self.tasks[task.tag] = task.get_tasks()
|
|
|
|
elif isinstance(tasks, spider.SpiderTask):
|
|
self.type = "single_spider"
|
|
self.tasks = {tasks.tag: task.get_tasks()}
|
|
else:
|
|
raise TypeError
|
|
|
|
async def start(self):
|
|
self.parseConnector.start_parse()
|
|
for r in asyncio.as_completed(self.tasks):
|
|
completed = await r
|
|
await self.parseConnector.add(
|
|
FrontItem(completed.type, completed.content, completed.cb)
|
|
)
|