Merge branch 'collector_11_11_liu' into dev

app/JsonRpc/CollectorService.php

@@ -1,17 +1,22 @@
 <?php
 namespace App\JsonRpc;
 
+use App\Model\ArticleData;
 use App\Model\OldModel\Article as OldArticle;
 use App\Model\Article;
+use App\Model\Rule;
 use App\Model\Web;
 use Hyperf\DbConnection\Db;
 use Hyperf\RpcServer\Annotation\RpcService;
 use App\Tools\Result;
+use QL\QueryList;
+use Swoole\Coroutine;
 
 
 #[RpcService(name: "CollectorService", protocol: "jsonrpc-http", server: "jsonrpc-http")]
 class CollectorService implements CollectorServiceInterface
 {
+
     /**
      * 添加网站
      * @param array $data
@@ -109,12 +114,232 @@ class CollectorService implements CollectorServiceInterface
      */
     public function sendCrawler(array $data): array
     {
-        $result =  Article::get();
-        $b = OldArticle::get();
-        $a = [
-            'old'=>$b,
-            'new'=>$result
+        //通过规则id 查询规则类型
+        $where = [
+            'rule.id'=>$data['id']
+        ];
+        $info = Rule::where($where)->leftJoin('web','rule.web_id','web.id')
+            ->select("rule.*","web.name as web_name","web.url as web_url","web.type as web_type")
+            ->first();
+        $info = $info->toArray();
+
+        switch ($info['web_type']){
+            case 1:
+                var_dump("wojinlailaile======",$info);
+                Rule::where(['id'=>$data['id']])->update(['status'=>1]);
+                $data['copyfrom'] = $info['web_name'];
+                $data['author'] = '刘德华';
+                $data['first_url'] = $info['first_url'];
+                $data['second_start'] = $info['second_start'];
+                $data['second_num'] = $info['second_num'];
+                $data['second_end'] = $info['second_end'];
+                $data['end_pagenum']= $info['end_pagenum'];
+                $data['rule_id']= $data['id'];
+                $data['admin_user_id']= $data['admin_user_id'];
+//                $data['newUrlStr'] =
+                $urlList = $this->addUrlArr($data);
+                if($urlList){
+                    foreach ($urlList as $val){
+                        var_dump("单列表地址：",$val);
+                        $this->ruleCollection($val,$data);
+                    }
+                }
+                Rule::where(['id'=>$data['id']])->update(['status'=>2]);
+                break;
+            case 2:
+
+                $wecUrl = $info['first_url'];//'https://www.ndcpa.gov.cn/queryList';
+                $parames = json_decode($info['parameter'],true);
+                $parames['webSiteCode'] = [trim($parames['webSiteCode'], "[]")]; //['jbkzzx'];//
+                $parames['channelCode'] =  [trim($parames['channelCode'], "[]")]; // ['c100008'];//
+                $other = [
+                    'web_url'=>$info['web_url'],
+                    'copyfrom'=>$info['web_name'],
+                    'admin_user_id'=>$data['admin_user_id'],
+                    'rule_id'=>$data['id']
+                ];
+                var_dump("开始调用接口方法====",$parames);
+//                die;
+                $this->foreachCurl($wecUrl,$parames,$other);
+
+        }
+        return  Result::success([]);
+    }
+
+    /**
+     * 把可采集的列表页连接 打包成一个大数组
+     * @return void
+     */
+    public function addUrlArr($data)
+    {
+        $arrList = [];
+        array_push($arrList,$data['first_url']);
+
+            $exit = false;
+            $i = 0;
+            while(!$exit){
+                $i++;
+                $url = $data['second_start'].$i.$data['second_end'];
+                $respon1 = Result::pageExists($url);
+                var_dump("采集地址：",$respon1,$url);
+//                Coroutine::sleep(2);
+                if ($i==intval($data['end_pagenum'])-1) {
+                    $exit = true;
+//                    Coroutine::exit(); // 退出循环
+                }else{
+                    array_push($arrList,$url);
+                }
+            }
+
+
+        return $arrList;
+    }
+
+    /**
+     * 按照规则采集数据
+     * @return void
+     */
+    public function ruleCollection($url,$data)
+    {
+        var_dump("采集参数：",$data);
+        $list = QueryList::get($url);
+        $dataList = $list->rules([
+            'title' => ['a', 'text'],
+            'link'  => ['a', 'href'],
+        ])->range('.list1 li')->query()->getData();
+        var_dump("采集的内容：",$dataList);
+//        var_dump("====",$dataList);die;
+        $firstUrlArr =  explode("/", $url);
+        array_pop($firstUrlArr);
+        $firstUrlArr = implode('/',$firstUrlArr);
+
+        $dataList = $dataList->toArray();
+        if($dataList){
+            foreach ($dataList as $tiem){
+                $newUrl =  substr($tiem['link'], 1);
+                $newUrlStr = $firstUrlArr.$newUrl;
+                $detailContent = QueryList::get($newUrlStr);
+                $detailData = $detailContent->rules([
+                    'title'=>['h1','text'],
+                    'content'=>['.TRS_UEDITOR','html'],
+                ])->range(".news-details")->query()->getData();
+
+                $detailData = $detailData->toArray();
+                var_dump("内容详情：",$detailData,$newUrlStr);
+                if($detailData){
+                    foreach ($detailData as $val){
+                        var_dump("进没进foreach:",$newUrlStr,$val);
+                        $data['fromurl'] = $newUrlStr;
+                        $data['title'] = $val['title'];
+                        $data['content'] = $val['content'];
+                        $data['newUrlStr'] = $newUrlStr;
+                        $data['source'] = '';
+                        $data['introduce'] = $val['title']??'';
+                        $data['keyword'] = $val['title']??'';
+                        $data['copyfrom'] = $data['copyfrom'];
+                        $data['source'] = $data['source']??$data['copyfrom'];
+                        $data['admin_user_id'] = $data['admin_user_id']??'';
+                        $data['rule_id'] = $data['rule_id']??'';
+//                        $data['copyfrom'] = $data['copyfrom'];
+                        var_dump("要插入的数据：",$data);
+                        $this->insertArticleData($data);
+                    }
+                }
+
+            }
+//
+        }
+    }
+
+    /**
+     * 插入数据
+     * @param $data
+     * @return void
+     */
+    public function insertArticleData($data=[])
+    {
+        if($data){
+            Db::beginTransaction();
+            try{
+                $articleInfo =  Article::where(['title'=>$data['title']])->first();
+                var_dump("获取详情：",$articleInfo,$data);
+                if(empty($articleInfo)){
+                    $insertData = [];
+                    $insertData['fromurl'] =$data['newUrlStr'];
+                    $insertData['oldtitle'] =$data['title'];
+                    $insertData['title'] = $data['title'];
+                    $insertData['copyfrom'] =  $data['copyfrom'];
+                    $insertData['author'] = $data['author'];
+                    $insertData['introduce'] = $data['title'];
+                    $insertData['keyword'] = $data['title'];
+                    $insertData['source'] = isset($data['source']) && $data['source']!=''? $data['source']:$data['copyfrom'];
+                    $insertData['admin_user_id'] = $data['admin_user_id'];
+                    $insertData['rule_id'] = $data['rule_id'];
+//                    var_dump("插入Article：",$insertData);
+                    $article_id = Article::insertGetId($insertData);
+                    $insertDataDetail = [];
+                    $insertDataDetail['article_id'] = $article_id;
+                    $insertDataDetail['content'] = $data['content'];
+                    var_dump("插入ArticleData：",$insertDataDetail);
+                    ArticleData::insertGetId($insertDataDetail);
+//                        Coroutine::sleep(2);
+//                    var_dump("插入成功一次：",$article_id,$insertDataDetail);
+                }
+                Db::commit();
+            }catch (\Exception $e){
+                Db::rollBack();
+                var_dump("插入失败：",$e->getMessage());
+            }
+
+        }else{
+            var_dump("没有数据可以插入：");
+        }
+
+    }
+
+    /**
+     * 分页采集
+     * @return void
+     */
+    public function foreachCurl($wecUrl,$parames,$other,&$page=1)
+    {
+        $options = [
+            CURLOPT_HEADER => true, // 如果想包含头部信息在响应中，可以设置为true
+            CURLOPT_TIMEOUT => 30 // 设置请求超时时间为30秒
         ];
-        return  Result::success($a);
+        $result = Result::http_post($wecUrl,$parames,$options);
+        $result = json_decode($result['response'],true);
+        var_dump("获取数据：",$result);
+        if($result['data'] && $result['data']['results']){
+            $dataList  = $result['data']['results'];
+//            var_dump("取数据结构体：",$dataList);
+            foreach ($dataList as $val){
+//                var_dump("进入循环插入：",$val);
+                $newUrlStr = json_decode($val['source']['urls'],true);
+                $newUrlStr = $other['web_url'].$newUrlStr['common'];
+//                var_dump("来源地址：",$newUrlStr);
+                $insertData = [
+                    'newUrlStr'=>$newUrlStr,
+                    'title'=>$val['source']['title']??'',
+                    'source'=>$val['source']['contentSource']??'',
+                    'copyfrom'=>$other['copyfrom']??'',
+                    'content'=>$val['source']['content']['content']??'',
+                    'admin_user_id'=>$other['admin_user_id']??'',
+                    'rule_id'=>$other['rule_id']??'',
+                    'author'=>'冯蕊'
+                ];
+//                var_dump("调用插入数据方法,组装数据：",$insertData);
+                $this->insertArticleData($insertData);
+            }
+        }
+        $pages = intval($parames['current']);
+        $pages =  $pages+1;
+        $parames['current'] = $pages;
+        $twoResult = Result::http_post($wecUrl,$parames,$options);
+        if($result['data'] && $result['data']['results'] && count($result['data']['results'])>0){
+            var_dump("分页测试：",$parames,$parames['current']);
+            $this->foreachCurl($wecUrl,$parames,$other,$pages);
+        }
+//        var_dump("正确的数据：",$result);
     }
 }

app/Model/ArticleData.php

@@ -0,0 +1,27 @@
+<?php
+
+declare(strict_types=1);
+
+namespace App\Model;
+
+use Hyperf\DbConnection\Model\Model;
+
+/**
+ */
+class ArticleData extends Model
+{
+    /**
+     * The table associated with the model.
+     */
+    protected ?string $table = 'article_data';
+
+    /**
+     * The attributes that are mass assignable.
+     */
+    protected array $fillable = [];
+
+    /**
+     * The attributes that should be cast to native types.
+     */
+    protected array $casts = [];
+}

app/Model/Rule.php

@@ -0,0 +1,27 @@
+<?php
+
+declare(strict_types=1);
+
+namespace App\Model;
+
+use Hyperf\DbConnection\Model\Model;
+
+/**
+ */
+class Rule extends Model
+{
+    /**
+     * The table associated with the model.
+     */
+    protected ?string $table = 'rule';
+
+    /**
+     * The attributes that are mass assignable.
+     */
+    protected array $fillable = [];
+
+    /**
+     * The attributes that should be cast to native types.
+     */
+    protected array $casts = [];
+}

app/Service/QueueService.php

@@ -0,0 +1,32 @@
+<?php
+
+declare(strict_types=1);
+
+namespace App\Service;
+
+use App\Job\ExampleJob;
+use Hyperf\AsyncQueue\Driver\DriverFactory;
+use Hyperf\AsyncQueue\Driver\DriverInterface;
+
+class QueueService
+{
+    protected DriverInterface $driver;
+
+    public function __construct(DriverFactory $driverFactory)
+    {
+        $this->driver = $driverFactory->get('default');
+    }
+
+    /**
+     * 生产消息.
+     * @param $params 数据
+     * @param int $delay 延时时间 单位秒
+     */
+    public function push($params, int $delay = 0): bool
+    {
+        // 这里的 `ExampleJob` 会被序列化存到 Redis 中，所以内部变量最好只传入普通数据
+        // 同理，如果内部使用了注解 @Value 会把对应对象一起序列化，导致消息体变大。
+        // 所以这里也不推荐使用 `make` 方法来创建 `Job` 对象。
+        return $this->driver->push(new ExampleJob($params), $delay);
+    }
+}

app/Tools/Result.php

@@ -23,4 +23,115 @@ class Result
             'data' => $data,
         ];
     }
+
+    //判断连接是否正确
+    public static function pageExists($url) {
+        $ch = curl_init();
+
+        // 设置 cURL 选项
+        curl_setopt($ch, CURLOPT_URL, $url);
+        curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); // 返回响应作为字符串，而不是直接输出
+        curl_setopt($ch, CURLOPT_NOBODY, false); // 如果你想检查内容，设置为 false
+        curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true); // 允许重定向
+        curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 100); // 设置连接超时时间
+        curl_setopt($ch, CURLOPT_TIMEOUT, 200); // 设置读取数据的超时时间
+        curl_setopt($ch, CURLOPT_FAILONERROR, true); // 在发生错误时返回 FALSE
+        curl_setopt($ch, CURLOPT_USERAGENT, 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'); // 设置用户代理，模拟浏览器请求
+
+        // 执行 cURL 请求
+        $response = curl_exec($ch);
+
+        // 检查是否有 cURL 错误发生
+        if (curl_errno($ch)) {
+            curl_close($ch);
+            return false;
+        }
+
+        // 获取 HTTP 状态码
+        $httpCode = curl_getinfo($ch, CURLINFO_HTTP_CODE);
+
+        // 关闭 cURL
+        curl_close($ch);
+
+        // 判断 HTTP 状态码是否在 200-299 范围内（表示成功）
+        // 或者你可以根据需要调整这个范围，比如包括 301, 302 重定向等
+        return $httpCode;
+    }
+
+    public static function http_get($url)
+    {
+        $oCurl = curl_init();
+        if (stripos($url, "https://") !== false) {
+            curl_setopt($oCurl, CURLOPT_SSL_VERIFYPEER, false);
+            curl_setopt($oCurl, CURLOPT_SSL_VERIFYHOST, false);
+            curl_setopt($oCurl, CURLOPT_SSLVERSION, 1); //CURL_SSLVERSION_TLSv1
+        }
+        curl_setopt($oCurl, CURLOPT_URL, $url);
+        curl_setopt($oCurl, CURLOPT_RETURNTRANSFER, 1);
+        $sContent = curl_exec($oCurl);
+        $aStatus = curl_getinfo($oCurl);
+        curl_close($oCurl);
+        if (intval($aStatus["http_code"]) == 200) {
+            return $sContent;
+        } else {
+            return false;
+        }
+    }
+
+    /**
+     * POST 请求
+     * @param string $url
+     * @param array $param
+     * @param boolean $post_file 是否文件上传
+     * @return string content
+     */
+    public static function http_post($url, $data, $options = [])
+    {
+
+        // 初始化CURL会话
+        $ch = curl_init($url);
+
+        // 设置CURL选项
+        curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); // 将curl_exec()获取的信息以文件流的形式返回，而不是直接输出。
+        curl_setopt($ch, CURLOPT_HEADER, false); // 将curl_exec()获取的信息以文件流的形式返回，而不是直接输出。
+        curl_setopt($ch, CURLOPT_POST, true); // 发送一个常规的POST请求。
+        curl_setopt($ch, CURLOPT_POSTFIELDS, http_build_query($data)); // 要传递的POST数据，这里使用http_build_query将数组转换为URL编码的查询字符串。
+
+        // 如果有额外的CURL选项，则合并它们
+        if (!empty($options)) {
+            curl_setopt_array($ch, $options);
+        }
+
+        // 执行CURL会话并获取响应
+        $response = curl_exec($ch);
+        // 检查是否有CURL错误
+        if (curl_errno($ch)) {
+            $error_msg = curl_error($ch);
+            curl_close($ch);
+            throw new Exception("CURL Error: $error_msg");
+        }
+
+        // 获取HTTP状态码
+        $http_code = curl_getinfo($ch, CURLINFO_HTTP_CODE);
+
+        // 关闭CURL会话
+        curl_close($ch);
+
+        // 返回一个包含响应和HTTP状态码的数组
+        $responseBody = $response;
+        $headerEnd = strpos($response, "\r\n\r\n");
+
+        if ($headerEnd !== false) {
+            // 去除响应头，只保留响应体
+            $responseBody = substr($response, $headerEnd + 4); // +4 是因为 "\r\n\r\n" 有4个字符
+//            echo $responseBody; // 输出：This is the response body.
+        } else {
+            // 如果没有找到空行，可能响应格式不正确或没有响应头
+//            echo "No headers found in response.";
+        }
+        return [
+            'response' => $responseBody,
+            'http_code' => $http_code
+        ];
+    }
 }

composer.json

@@ -14,6 +14,7 @@
     "require": {
         "php": ">=8.1",
         "doctrine/annotations": "^2.0",
+        "hyperf/async-queue": "^3.1",
         "hyperf/cache": "~3.1.0",
         "hyperf/command": "~3.1.0",
         "hyperf/config": "~3.1.0",
@@ -36,7 +37,8 @@
         "hyperf/service-governance": "^3.1",
         "hyperf/service-governance-consul": "^3.1",
         "hyperf/service-governance-nacos": "^3.1",
-        "hyperf/utils": "^3.1"
+        "hyperf/utils": "^3.1",
+        "jaeger/querylist": "^4.4"
     },
     "require-dev": {
         "friendsofphp/php-cs-fixer": "^3.0",

composer.lock

@@ -4,7 +4,7 @@
         "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
         "This file is @generated automatically"
     ],
-    "content-hash": "1935bfafe4523350f25d98a64e75d710",
+    "content-hash": "b6123e07a5d46021d034b563b38afc91",
     "packages": [
         {
             "name": "carbonphp/carbon-doctrine-types",
@@ -832,6 +832,83 @@
             ],
             "time": "2024-07-18T11:15:46+00:00"
         },
+        {
+            "name": "hyperf/async-queue",
+            "version": "v3.1.42",
+            "source": {
+                "type": "git",
+                "url": "https://github.com/hyperf/async-queue.git",
+                "reference": "1cd25666ac1e1f23c9eab6be642e86802a96307b"
+            },
+            "dist": {
+                "type": "zip",
+                "url": "https://api.github.com/repos/hyperf/async-queue/zipball/1cd25666ac1e1f23c9eab6be642e86802a96307b",
+                "reference": "1cd25666ac1e1f23c9eab6be642e86802a96307b",
+                "shasum": ""
+            },
+            "require": {
+                "hyperf/codec": "~3.1.0",
+                "hyperf/collection": "~3.1.0",
+                "hyperf/command": "~3.1.0",
+                "hyperf/contract": "~3.1.0",
+                "hyperf/support": "~3.1.0",
+                "hyperf/utils": "~3.1.0",
+                "php": ">=8.1",
+                "psr/container": "^1.0 || ^2.0",
+                "psr/event-dispatcher": "^1.0"
+            },
+            "suggest": {
+                "hyperf/di": "Required to use annotations.",
+                "hyperf/event": "Required to dispatch a event.",
+                "hyperf/logger": "Required to use QueueHandleListener.",
+                "hyperf/process": "Auto register the consumer process for server."
+            },
+            "type": "library",
+            "extra": {
+                "branch-alias": {
+                    "dev-master": "3.1-dev"
+                },
+                "hyperf": {
+                    "config": "Hyperf\\AsyncQueue\\ConfigProvider"
+                }
+            },
+            "autoload": {
+                "files": [
+                    "src/Functions.php"
+                ],
+                "psr-4": {
+                    "Hyperf\\AsyncQueue\\": "src/"
+                }
+            },
+            "notification-url": "https://packagist.org/downloads/",
+            "license": [
+                "MIT"
+            ],
+            "description": "A async queue component for hyperf.",
+            "homepage": "https://hyperf.io",
+            "keywords": [
+                "async-queue",
+                "hyperf",
+                "php"
+            ],
+            "support": {
+                "docs": "https://hyperf.wiki",
+                "issues": "https://github.com/hyperf/hyperf/issues",
+                "pull-request": "https://github.com/hyperf/hyperf/pulls",
+                "source": "https://github.com/hyperf/hyperf"
+            },
+            "funding": [
+                {
+                    "url": "https://hyperf.wiki/#/zh-cn/donate",
+                    "type": "custom"
+                },
+                {
+                    "url": "https://opencollective.com/hyperf",
+                    "type": "open_collective"
+                }
+            ],
+            "time": "2024-09-25T02:54:12+00:00"
+        },
         {
             "name": "hyperf/cache",
             "version": "v3.1.43",
@@ -4215,6 +4292,161 @@
             ],
             "time": "2024-09-25T02:54:12+00:00"
         },
+        {
+            "name": "jaeger/g-http",
+            "version": "V2.0.1",
+            "source": {
+                "type": "git",
+                "url": "https://github.com/jae-jae/GHttp.git",
+                "reference": "29cb537c57f230980ca10622de190f04831e35e6"
+            },
+            "dist": {
+                "type": "zip",
+                "url": "https://api.github.com/repos/jae-jae/GHttp/zipball/29cb537c57f230980ca10622de190f04831e35e6",
+                "reference": "29cb537c57f230980ca10622de190f04831e35e6",
+                "shasum": ""
+            },
+            "require": {
+                "guzzlehttp/guzzle": "^6.0 || ^7.0",
+                "php": ">=8.1",
+                "symfony/cache": "^6.4 || ^7.0"
+            },
+            "require-dev": {
+                "predis/predis": "^2.2"
+            },
+            "type": "library",
+            "autoload": {
+                "psr-4": {
+                    "Jaeger\\": "src"
+                }
+            },
+            "notification-url": "https://packagist.org/downloads/",
+            "license": [
+                "MIT"
+            ],
+            "authors": [
+                {
+                    "name": "Jaeger",
+                    "email": "JaegerCode@gmail.com"
+                }
+            ],
+            "description": "Simple Http client base on GuzzleHttp",
+            "support": {
+                "issues": "https://github.com/jae-jae/GHttp/issues",
+                "source": "https://github.com/jae-jae/GHttp/tree/V2.0.1"
+            },
+            "time": "2024-06-14T08:13:32+00:00"
+        },
+        {
+            "name": "jaeger/phpquery-single",
+            "version": "1.1.1",
+            "source": {
+                "type": "git",
+                "url": "https://github.com/jae-jae/phpQuery-single.git",
+                "reference": "39a650ade692a6b480c22220dce0c198d6a946fb"
+            },
+            "dist": {
+                "type": "zip",
+                "url": "https://api.github.com/repos/jae-jae/phpQuery-single/zipball/39a650ade692a6b480c22220dce0c198d6a946fb",
+                "reference": "39a650ade692a6b480c22220dce0c198d6a946fb",
+                "shasum": ""
+            },
+            "require": {
+                "php": ">=5.3.0"
+            },
+            "type": "library",
+            "autoload": {
+                "classmap": [
+                    "phpQuery.php"
+                ]
+            },
+            "notification-url": "https://packagist.org/downloads/",
+            "license": [
+                "MIT"
+            ],
+            "authors": [
+                {
+                    "name": "Tobiasz Cudnik",
+                    "email": "tobiasz.cudnik@gmail.com",
+                    "homepage": "https://github.com/TobiaszCudnik",
+                    "role": "Developer"
+                },
+                {
+                    "name": "Jaeger",
+                    "role": "Packager"
+                }
+            ],
+            "description": "phpQuery单文件版本，是Querylist的依赖(http://querylist.cc/)，phpQuery项目主页:http://code.google.com/p/phpquery/",
+            "homepage": "http://code.google.com/p/phpquery/",
+            "support": {
+                "issues": "https://github.com/jae-jae/phpQuery-single/issues",
+                "source": "https://github.com/jae-jae/phpQuery-single/tree/1.1.1"
+            },
+            "time": "2022-03-26T15:01:16+00:00"
+        },
+        {
+            "name": "jaeger/querylist",
+            "version": "V4.4.5",
+            "source": {
+                "type": "git",
+                "url": "https://github.com/jae-jae/QueryList.git",
+                "reference": "87b405ecde30101ec8797c4347f05f9ee0b95eb2"
+            },
+            "dist": {
+                "type": "zip",
+                "url": "https://api.github.com/repos/jae-jae/QueryList/zipball/87b405ecde30101ec8797c4347f05f9ee0b95eb2",
+                "reference": "87b405ecde30101ec8797c4347f05f9ee0b95eb2",
+                "shasum": ""
+            },
+            "require": {
+                "ext-dom": "*",
+                "jaeger/g-http": "^2.0",
+                "jaeger/phpquery-single": "^1",
+                "php": ">=8.1",
+                "symfony/var-dumper": ">3.4"
+            },
+            "require-dev": {
+                "phpunit/phpunit": "^8.5"
+            },
+            "type": "library",
+            "autoload": {
+                "files": [
+                    "src/Collect/Support/helpers.php",
+                    "src/Collect/Support/alias.php"
+                ],
+                "psr-4": {
+                    "QL\\": "src"
+                }
+            },
+            "notification-url": "https://packagist.org/downloads/",
+            "license": [
+                "MIT"
+            ],
+            "authors": [
+                {
+                    "name": "Jaeger",
+                    "email": "JaegerCode@gmail.com"
+                }
+            ],
+            "description": "Simple, elegant, extensible PHP Web Scraper (crawler/spider),Use the css3 dom selector,Based on phpQuery! 简洁、优雅、可扩展的PHP采集工具(爬虫)，基于phpQuery。",
+            "homepage": "http://querylist.cc",
+            "keywords": [
+                "QueryList",
+                "phpQuery",
+                "spider"
+            ],
+            "support": {
+                "issues": "https://github.com/jae-jae/QueryList/issues",
+                "source": "https://github.com/jae-jae/QueryList/tree/V4.4.5"
+            },
+            "funding": [
+                {
+                    "url": "https://opencollective.com/querylist",
+                    "type": "open_collective"
+                }
+            ],
+            "time": "2024-07-16T07:11:08+00:00"
+        },
         {
             "name": "jetbrains/phpstorm-attributes",
             "version": "1.1",
@@ -5551,6 +5783,178 @@
             },
             "time": "2023-06-15T09:18:11+00:00"
         },
+        {
+            "name": "symfony/cache",
+            "version": "v6.4.14",
+            "source": {
+                "type": "git",
+                "url": "https://github.com/symfony/cache.git",
+                "reference": "36fb8aa88833708e9f29014b6f15fac051a8b613"
+            },
+            "dist": {
+                "type": "zip",
+                "url": "https://api.github.com/repos/symfony/cache/zipball/36fb8aa88833708e9f29014b6f15fac051a8b613",
+                "reference": "36fb8aa88833708e9f29014b6f15fac051a8b613",
+                "shasum": ""
+            },
+            "require": {
+                "php": ">=8.1",
+                "psr/cache": "^2.0|^3.0",
+                "psr/log": "^1.1|^2|^3",
+                "symfony/cache-contracts": "^2.5|^3",
+                "symfony/service-contracts": "^2.5|^3",
+                "symfony/var-exporter": "^6.3.6|^7.0"
+            },
+            "conflict": {
+                "doctrine/dbal": "<2.13.1",
+                "symfony/dependency-injection": "<5.4",
+                "symfony/http-kernel": "<5.4",
+                "symfony/var-dumper": "<5.4"
+            },
+            "provide": {
+                "psr/cache-implementation": "2.0|3.0",
+                "psr/simple-cache-implementation": "1.0|2.0|3.0",
+                "symfony/cache-implementation": "1.1|2.0|3.0"
+            },
+            "require-dev": {
+                "cache/integration-tests": "dev-master",
+                "doctrine/dbal": "^2.13.1|^3|^4",
+                "predis/predis": "^1.1|^2.0",
+                "psr/simple-cache": "^1.0|^2.0|^3.0",
+                "symfony/config": "^5.4|^6.0|^7.0",
+                "symfony/dependency-injection": "^5.4|^6.0|^7.0",
+                "symfony/filesystem": "^5.4|^6.0|^7.0",
+                "symfony/http-kernel": "^5.4|^6.0|^7.0",
+                "symfony/messenger": "^5.4|^6.0|^7.0",
+                "symfony/var-dumper": "^5.4|^6.0|^7.0"
+            },
+            "type": "library",
+            "autoload": {
+                "psr-4": {
+                    "Symfony\\Component\\Cache\\": ""
+                },
+                "classmap": [
+                    "Traits/ValueWrapper.php"
+                ],
+                "exclude-from-classmap": [
+                    "/Tests/"
+                ]
+            },
+            "notification-url": "https://packagist.org/downloads/",
+            "license": [
+                "MIT"
+            ],
+            "authors": [
+                {
+                    "name": "Nicolas Grekas",
+                    "email": "p@tchwork.com"
+                },
+                {
+                    "name": "Symfony Community",
+                    "homepage": "https://symfony.com/contributors"
+                }
+            ],
+            "description": "Provides extended PSR-6, PSR-16 (and tags) implementations",
+            "homepage": "https://symfony.com",
+            "keywords": [
+                "caching",
+                "psr6"
+            ],
+            "support": {
+                "source": "https://github.com/symfony/cache/tree/v6.4.14"
+            },
+            "funding": [
+                {
+                    "url": "https://symfony.com/sponsor",
+                    "type": "custom"
+                },
+                {
+                    "url": "https://github.com/fabpot",
+                    "type": "github"
+                },
+                {
+                    "url": "https://tidelift.com/funding/github/packagist/symfony/symfony",
+                    "type": "tidelift"
+                }
+            ],
+            "time": "2024-11-05T15:34:40+00:00"
+        },
+        {
+            "name": "symfony/cache-contracts",
+            "version": "v3.5.0",
+            "source": {
+                "type": "git",
+                "url": "https://github.com/symfony/cache-contracts.git",
+                "reference": "df6a1a44c890faded49a5fca33c2d5c5fd3c2197"
+            },
+            "dist": {
+                "type": "zip",
+                "url": "https://api.github.com/repos/symfony/cache-contracts/zipball/df6a1a44c890faded49a5fca33c2d5c5fd3c2197",
+                "reference": "df6a1a44c890faded49a5fca33c2d5c5fd3c2197",
+                "shasum": ""
+            },
+            "require": {
+                "php": ">=8.1",
+                "psr/cache": "^3.0"
+            },
+            "type": "library",
+            "extra": {
+                "branch-alias": {
+                    "dev-main": "3.5-dev"
+                },
+                "thanks": {
+                    "name": "symfony/contracts",
+                    "url": "https://github.com/symfony/contracts"
+                }
+            },
+            "autoload": {
+                "psr-4": {
+                    "Symfony\\Contracts\\Cache\\": ""
+                }
+            },
+            "notification-url": "https://packagist.org/downloads/",
+            "license": [
+                "MIT"
+            ],
+            "authors": [
+                {
+                    "name": "Nicolas Grekas",
+                    "email": "p@tchwork.com"
+                },
+                {
+                    "name": "Symfony Community",
+                    "homepage": "https://symfony.com/contributors"
+                }
+            ],
+            "description": "Generic abstractions related to caching",
+            "homepage": "https://symfony.com",
+            "keywords": [
+                "abstractions",
+                "contracts",
+                "decoupling",
+                "interfaces",
+                "interoperability",
+                "standards"
+            ],
+            "support": {
+                "source": "https://github.com/symfony/cache-contracts/tree/v3.5.0"
+            },
+            "funding": [
+                {
+                    "url": "https://symfony.com/sponsor",
+                    "type": "custom"
+                },
+                {
+                    "url": "https://github.com/fabpot",
+                    "type": "github"
+                },
+                {
+                    "url": "https://tidelift.com/funding/github/packagist/symfony/symfony",
+                    "type": "tidelift"
+                }
+            ],
+            "time": "2024-04-18T09:32:20+00:00"
+        },
         {
             "name": "symfony/console",
             "version": "v6.4.14",
@@ -6516,6 +6920,168 @@
             ],
             "time": "2024-04-18T09:32:20+00:00"
         },
+        {
+            "name": "symfony/var-dumper",
+            "version": "v6.4.14",
+            "source": {
+                "type": "git",
+                "url": "https://github.com/symfony/var-dumper.git",
+                "reference": "93c09246038178717a9c14b809ea8151ffcf7091"
+            },
+            "dist": {
+                "type": "zip",
+                "url": "https://api.github.com/repos/symfony/var-dumper/zipball/93c09246038178717a9c14b809ea8151ffcf7091",
+                "reference": "93c09246038178717a9c14b809ea8151ffcf7091",
+                "shasum": ""
+            },
+            "require": {
+                "php": ">=8.1",
+                "symfony/deprecation-contracts": "^2.5|^3",
+                "symfony/polyfill-mbstring": "~1.0"
+            },
+            "conflict": {
+                "symfony/console": "<5.4"
+            },
+            "require-dev": {
+                "ext-iconv": "*",
+                "symfony/console": "^5.4|^6.0|^7.0",
+                "symfony/error-handler": "^6.3|^7.0",
+                "symfony/http-kernel": "^5.4|^6.0|^7.0",
+                "symfony/process": "^5.4|^6.0|^7.0",
+                "symfony/uid": "^5.4|^6.0|^7.0",
+                "twig/twig": "^2.13|^3.0.4"
+            },
+            "bin": [
+                "Resources/bin/var-dump-server"
+            ],
+            "type": "library",
+            "autoload": {
+                "files": [
+                    "Resources/functions/dump.php"
+                ],
+                "psr-4": {
+                    "Symfony\\Component\\VarDumper\\": ""
+                },
+                "exclude-from-classmap": [
+                    "/Tests/"
+                ]
+            },
+            "notification-url": "https://packagist.org/downloads/",
+            "license": [
+                "MIT"
+            ],
+            "authors": [
+                {
+                    "name": "Nicolas Grekas",
+                    "email": "p@tchwork.com"
+                },
+                {
+                    "name": "Symfony Community",
+                    "homepage": "https://symfony.com/contributors"
+                }
+            ],
+            "description": "Provides mechanisms for walking through any arbitrary PHP variable",
+            "homepage": "https://symfony.com",
+            "keywords": [
+                "debug",
+                "dump"
+            ],
+            "support": {
+                "source": "https://github.com/symfony/var-dumper/tree/v6.4.14"
+            },
+            "funding": [
+                {
+                    "url": "https://symfony.com/sponsor",
+                    "type": "custom"
+                },
+                {
+                    "url": "https://github.com/fabpot",
+                    "type": "github"
+                },
+                {
+                    "url": "https://tidelift.com/funding/github/packagist/symfony/symfony",
+                    "type": "tidelift"
+                }
+            ],
+            "time": "2024-11-05T15:34:40+00:00"
+        },
+        {
+            "name": "symfony/var-exporter",
+            "version": "v6.4.13",
+            "source": {
+                "type": "git",
+                "url": "https://github.com/symfony/var-exporter.git",
+                "reference": "0f605f72a363f8743001038a176eeb2a11223b51"
+            },
+            "dist": {
+                "type": "zip",
+                "url": "https://api.github.com/repos/symfony/var-exporter/zipball/0f605f72a363f8743001038a176eeb2a11223b51",
+                "reference": "0f605f72a363f8743001038a176eeb2a11223b51",
+                "shasum": ""
+            },
+            "require": {
+                "php": ">=8.1",
+                "symfony/deprecation-contracts": "^2.5|^3"
+            },
+            "require-dev": {
+                "symfony/property-access": "^6.4|^7.0",
+                "symfony/serializer": "^6.4|^7.0",
+                "symfony/var-dumper": "^5.4|^6.0|^7.0"
+            },
+            "type": "library",
+            "autoload": {
+                "psr-4": {
+                    "Symfony\\Component\\VarExporter\\": ""
+                },
+                "exclude-from-classmap": [
+                    "/Tests/"
+                ]
+            },
+            "notification-url": "https://packagist.org/downloads/",
+            "license": [
+                "MIT"
+            ],
+            "authors": [
+                {
+                    "name": "Nicolas Grekas",
+                    "email": "p@tchwork.com"
+                },
+                {
+                    "name": "Symfony Community",
+                    "homepage": "https://symfony.com/contributors"
+                }
+            ],
+            "description": "Allows exporting any serializable PHP data structure to plain PHP code",
+            "homepage": "https://symfony.com",
+            "keywords": [
+                "clone",
+                "construct",
+                "export",
+                "hydrate",
+                "instantiate",
+                "lazy-loading",
+                "proxy",
+                "serialize"
+            ],
+            "support": {
+                "source": "https://github.com/symfony/var-exporter/tree/v6.4.13"
+            },
+            "funding": [
+                {
+                    "url": "https://symfony.com/sponsor",
+                    "type": "custom"
+                },
+                {
+                    "url": "https://github.com/fabpot",
+                    "type": "github"
+                },
+                {
+                    "url": "https://tidelift.com/funding/github/packagist/symfony/symfony",
+                    "type": "tidelift"
+                }
+            ],
+            "time": "2024-09-25T14:18:03+00:00"
+        },
         {
             "name": "vlucas/phpdotenv",
             "version": "v5.6.1",
@@ -10272,12 +10838,12 @@
     ],
     "aliases": [],
     "minimum-stability": "dev",
-    "stability-flags": {},
+    "stability-flags": [],
     "prefer-stable": true,
     "prefer-lowest": false,
     "platform": {
         "php": ">=8.1"
     },
-    "platform-dev": {},
+    "platform-dev": [],
     "plugin-api-version": "2.6.0"
 }

config/autoload/async_queue.php

@@ -0,0 +1,30 @@
+<?php
+
+declare(strict_types=1);
+/**
+ * This file is part of Hyperf.
+ *
+ * @link     https://www.hyperf.io
+ * @document https://hyperf.wiki
+ * @contact  group@hyperf.io
+ * @license  https://github.com/hyperf/hyperf/blob/master/LICENSE
+ */
+use Hyperf\AsyncQueue\Driver\RedisDriver;
+
+return [
+    'default' => [
+        'driver' => RedisDriver::class,
+        'redis' => [
+            'pool' => 'default',
+        ],
+        'channel' => '{queue}',
+        'timeout' => 2,
+        'retry_seconds' => 5,
+        'handle_timeout' => 10,
+        'processes' => 1,
+        'concurrent' => [
+            'limit' => 10,
+        ],
+        'max_messages' => 0,
+    ],
+];

config/autoload/processes.php

@@ -10,4 +10,5 @@ declare(strict_types=1);
  * @license  https://github.com/hyperf/hyperf/blob/master/LICENSE
  */
 return [
+//    Hyperf\AsyncQueue\Process\ConsumerProcess::class, //异步消费进程
 ];

runtime/container/aspects.cache

@@ -1 +1 @@
-a:8:{i:0;s:35:"Hyperf\Cache\Aspect\CacheableAspect";i:1;s:36:"Hyperf\Cache\Aspect\CacheAheadAspect";i:2;s:36:"Hyperf\Cache\Aspect\CacheEvictAspect";i:3;s:34:"Hyperf\Cache\Aspect\CachePutAspect";i:4;s:35:"Hyperf\Cache\Aspect\FailCacheAspect";i:5;s:36:"Hyperf\Config\Annotation\ValueAspect";i:6;s:44:"Hyperf\DbConnection\Aspect\TransactionAspect";i:7;s:33:"Hyperf\Di\Annotation\InjectAspect";}
+a:9:{i:0;s:41:"Hyperf\AsyncQueue\Aspect\AsyncQueueAspect";i:1;s:35:"Hyperf\Cache\Aspect\CacheableAspect";i:2;s:36:"Hyperf\Cache\Aspect\CacheAheadAspect";i:3;s:36:"Hyperf\Cache\Aspect\CacheEvictAspect";i:4;s:34:"Hyperf\Cache\Aspect\CachePutAspect";i:5;s:35:"Hyperf\Cache\Aspect\FailCacheAspect";i:6;s:36:"Hyperf\Config\Annotation\ValueAspect";i:7;s:44:"Hyperf\DbConnection\Aspect\TransactionAspect";i:8;s:33:"Hyperf\Di\Annotation\InjectAspect";}

runtime/hyperf.pid

@@ -1 +1 @@
-249
+97083