采集器功能代码

app/JsonRpc/CollectorService.php

@@ -3,15 +3,20 @@ namespace App\JsonRpc;
 
 use App\Model\ArticleData;
 use App\Model\OldModel\Article as OldArticle;
+use App\Model\OldModel\ArticleData as OldArticleData;
+use App\Model\OldModel\Category;
 use App\Model\Article;
 use App\Model\Rule;
 use App\Model\Web;
+use App\Model\Rule;
+use App\Model\ArticleData;
 use Hyperf\DbConnection\Db;
 use Hyperf\RpcServer\Annotation\RpcService;
 use App\Tools\Result;
 use QL\QueryList;
 use Swoole\Coroutine;
 
+use function Hyperf\Support\retry;
 
 #[RpcService(name: "CollectorService", protocol: "jsonrpc-http", server: "jsonrpc-http")]
 class CollectorService implements CollectorServiceInterface
@@ -30,9 +35,6 @@ class CollectorService implements CollectorServiceInterface
         $isweb = Web::where($where)->first();
         if(empty($isweb)){
             date_default_timezone_set('Asia/Shanghai');
-            $time = time();
-            $catetime = date('Y-m-d H:i:s', $time);
-            $data['created_at'] = $catetime;
             $web = Web::insert($data);
             
         }else{
@@ -55,20 +57,26 @@ class CollectorService implements CollectorServiceInterface
             $where = [
                 ['name','like','%'.$data['keyWord'].'%']
             ];
-            $webss = Web::where($where)->first();
-            if(empty($webss)){
+            $rep = Web::where($where)->limit($data['pageSize'])->orderBy("created_at","desc")->offset(($data['page']-1)*$data['pageSize'])->get();
+            $count =  Web::where($where)->count();
+            if($count==0){
                 return Result::error('未查找到相关网站！');
             }
         }else{
-            $web = Web::get();
+            $rep = Web::limit($data['pageSize'])->orderBy("created_at","desc")->offset(($data['page']-1)*$data['pageSize'])->get();
+            $count =  Web::count();
+            
         }
-        
-        if(empty($web)){
+        $data = [
+            'rep' => $rep->toArray(),
+            'count' => $count
+        ];
+        if(empty($rep)){
             return Result::error('您还未添加网站，请先去添加！');
             
         }
         
-        return Result::success($web);
+        return Result::success($data);
     }
     /**
      * 修改网站
@@ -77,11 +85,13 @@ class CollectorService implements CollectorServiceInterface
      */
     public function upWeb(array $data): array
     {
+        
         $web = Web::where('id',$data['id'])->first();
         if(empty($web)){
             return Result::error('请输入正确的网站id！');
             
         }else{
+            date_default_timezone_set('Asia/Shanghai');
             $id = Web::where('id',$data['id'])->update($data);
             if(empty($id)){
                 return Result::error('无法修改！');
@@ -101,14 +111,227 @@ class CollectorService implements CollectorServiceInterface
             return Result::error('请输入正确的网站id！');
             
         }else{
-            $id = Web::where('id',$data['id'])->delete();
-            if(empty($id)){
-                return Result::error('无法删除！');
+            $where = [
+                ['web_id','=',$data['id']]
+            ];
+            //判断此网站下是否规则任务
+            $rule = Rule::where($where)->get();
+            if(empty($rule)){
+                //若没有直接删除网站
+                $result['web'] = Web::where('id',$data['id'])->delete();
+            }else{
+                //若有，判断规则任务是否有已执行的
+                $rule = Rule::where($where)->where('status',2)->get();
+                // return Result::success($rule);
+                if(!empty($rule->toArray())){
+                    //若有已执行的任务规则，不可删除网站
+                    return Result::error('该网站已有成功执行的任务规则，不可删除！');
+                }else{
+                    try {
+                        Db::beginTransaction();
+                        //若无已执行的任务规则，删除网站及相应的未执行的规则任务
+                        $result['web'] = Web::where('id',$data['id'])->delete();
+                        $result['rule'] = Rule::where($where)->delete();
+                        Db::commit();
+                    } catch(\Throwable $ex){
+                        Db::rollBack();
+                        var_dump($ex->getMessage());
+                        return Result::error("删除失败",0);
+                    }
+                }        
             }
         }
-        return Result::success($id);
+        return Result::success($result);
+    }
+    /**
+     * 添加任务规则
+     * @param array $data
+     * @return array|mixed
+     */
+    public function addRule(array $data): array
+    {
+        $web = Web::where('id',$data['web_id'])->get();
+        if(empty($web->toArray())){
+            return Result::error('请输入正确的网站id！');
+            
+        }else{
+            $rulename = Rule::where('name',$data['name'])->get();
+            //查找是否存在规则名称重复的
+            if(empty($rulename->toArray())){
+                //（若是多类型参数一起传过来则根据类型，只获取对应类型需要的参数）
+                switch($data['type']){
+                    case 1:
+                        $rule = [
+                            'name' => $data['name'],
+                            'web_id' => $data['web_id'],
+                            'first_url' => $data['first_url'],
+                            'second_start' => $data['second_start'],
+                            'second_num' => $data['second_num'],
+                            'second_end' => $data['second_end'],
+                            'end_pagenum' => $data['end_pagenum'],
+                            'start' => $data['start'],
+                            'title' => $data['title'],
+                            'content' => $data['content']
+                        ];
+                        // var_dump("============1============");       
+                        break;         
+                    case 2:
+                        $rule = [
+                            'name' => $data['name'],
+                            'web_id' => $data['web_id'],
+                            'first_url' => $data['first_url'],
+                            'parameter' => $data['parameter'],
+                            'start' => $data['start'],
+                            'title' => $data['title'],
+                            'content' => $data['content']
+                        ];  
+                        // var_dump("============2============");       
+                        break;        
+                    default:
+                        $rule = [
+                            'name' => $data['name'],
+                            'web_id' => $data['web_id'],
+                            'diy_rule' => $data['diy_rule']
+                        ];        
+                        // var_dump("============3============");       
+                        break;              
+                }
+                if(isset($data['source']) && $data['type'] != 3){
+                    $rule ['source'] = $data['source'];
+                }
+                if(isset($data['writer_class']) && $data['type'] != 3){
+                    $rule ['writer_class'] = $data['writer_class'];
+                }
+                if(isset($data['writer']) && $data['type'] != 3){
+                    $rule ['writer'] = $data['writer'];
+                }
+                date_default_timezone_set('Asia/Shanghai');
+                //若不存在，根据网站类型添加到不行类型的规则表中
+                $result = Rule::insertGetId($rule);
+                
+            }else{
+                return Result::error('此任务已存在！');
+            }  
+        }
+        return Result::success($result);
+    }
+    
+    /**
+     * 获取并搜索规则任务
+     * @param array $data
+     * @return array|mixed
+     */
+    public function getRule(array $data): array
+    {
+        if(isset($data['web_id'])){
+            $web = Web::where('id',$data['web_id'])->get();
+            if(empty($web->toArray())){
+                return Result::error('请输入正确的网站id!');
+                
+            }else{
+                $where = [
+                    ['web_id','=', $data['web_id']]
+                ];
+            }
+        } 
+        if(isset($data['keyWord'])){
+            //若存在搜索词,则存到条件数组$where中
+            $where = [
+                ['name','like','%'.$data['keyWord'].'%']
+            ];
+        }
+        $rep = Rule::withCount(relations:'arts')->where($where)->limit($data['pageSize'])->orderBy("created_at","desc")->offset(($data['page']-1)*$data['pageSize'])->get();
+        $count = Rule::where($where)->count();
+        if($count==0){
+            return Result::error('暂无相关规则任务！');
+        }       
+        
+        $data = [
+            'rep' => $rep->toArray(),
+            'count' => $count
+        ];
+        return Result::success($data);
+    }
+     /**
+     * 获取某个任务规则
+     * @param array $data
+     * @return array|mixed
+     */
+    public function getOneRule(array $data): array
+    {
+        $result = Rule::where('id',$data['id'])->first();
+        if(empty($result)){
+            return Result::error('请输入正确的规则任务id！');
+            
+        }else{
+            return Result::success($result);
+        } 
+    }
+    /**
+     * 修改规则任务
+     * @param array $data
+     * @return array|mixed
+     */
+    public function upRule(array $data): array
+    {
+        $rule = Rule::where('id',$data['id'])->select('id')->first();
+        unset($data['type']);
+        if(empty($rule)){
+            return Result::error('请输入正确的规则任务id！');
+            
+        }else{
+            $rulename = Rule::where('id','!=',$rule['id'])->where('name',$data['name'])->select('name')->first();
+            if(empty($rulename)){
+                $result = Rule::where('id',$data['id'])->update($data);
+            }else{
+                return Result::error('已存在此任务规则名称！');
+            }
+        }
+       
+        return Result::success($result);
+    }
+    /**
+     * 删除规则任务
+     * @param array $data
+     * @return array
+     */
+    public function delRule(array $data): array
+    {
+        $where = ['id' => $data['rule_id']];
+        $rule = Rule::where($where)->first();
+        if(empty($rule)){
+            return Result::error('请输入正确的规则任务id！');
+            
+        }else{
+            //查找是否存在已导入的文章
+            $art_num = Article::where('rule_id',$data['rule_id'])->where('state',1)->count();
+            if($art_num==0){
+                //查找是否存在已采集但是未导入的文章
+                $colart_num = Article::where('rule_id',$data['rule_id'])->where('state',0)->count();
+                if($colart_num==0){
+                    $result['rule'] = Rule::where($where)->delete();
+                }else{
+                    try {
+                        Db::beginTransaction();
+                        //若有已采集但未导入的文章，删除规则任务及相应的未导入的文章
+                        $result['rule'] = Rule::where($where)->delete();
+                        $result['art'] = Article::where('rule_id',$data['rule_id'])->delete();
+                        Db::commit();
+                    } catch(\Throwable $ex){
+                        Db::rollBack();
+                        var_dump($ex->getMessage());
+                        return Result::error("删除失败",0);
+                    }
+                }
+            }else{
+                return Result::error('此规则任务下的文章已导入，不可删除！');
+            }
+        }
+        return Result::success($result);
     }
+
     /**
+     * 开始采集
      * @param array $data
      * @return array
      */
@@ -342,4 +565,222 @@ class CollectorService implements CollectorServiceInterface
         }
 //        var_dump("正确的数据：",$result);
     }
+    /**
+     * 获取并搜索资讯
+     * @param array $data
+     * @return array
+     */
+    public function getInfo(array $data): array
+    {
+        $where = [
+            ['rule_id','=',$data['rule_id']]
+        ];
+        //若存在条件参数都存到where数组
+        if(isset($data['title'])){
+            $where[] = ['title','like','%'.$data['title'].'%'];
+        }
+        if(isset($data['source'])){
+            $art_source = Article::where($where)->get();
+            if(!empty($art_source->toArray())){
+                $where[] = ['source','=',$data['source']];
+            } 
+        }
+        if(isset($data['state'])){
+            $where[] = ['state','=',$data['state']];
+        }
+        //跨库查询栏目导航及采集的新闻
+        $info = Article::query()
+        ->where($where)
+        ->with(['category' => function ($query) {
+            $query->select('name');
+        }])
+        ->orderBy("article.id","desc")
+        ->limit($data['pageSize'])
+        ->offset(($data['page']-1)*$data['pageSize'])->get();
+        $count = Article::where($where)->count();
+        if($count == 0){
+            return Result::error('暂无资讯');
+        }
+        $data = [
+            'rep' => $info->toArray(),
+            'count' => $count
+        ];
+        return  Result::success($data);
+    }
+    /**
+     * 获取某个资讯
+     * @param array $data
+     * @return array
+     */
+    public function getOneInfo(array $data): array
+    {
+        $where = ['id' => $data['art_id']];
+        $inf = Article::where($where)->first();
+        if($inf==null){
+            return Result::error('请输入正确的资讯id！');
+        }
+        $info =  Article::where($where)
+        ->leftJoin('article_data','article_id','id')
+        ->select('article.*','article_data.content')
+        ->first();
+        if($inf['catid']!=null){
+            $category = Category::where(['id'=>$info['catid']])->select('name')->first();
+            $info['category'] = $category['name'];
+        }
+        
+        return  Result::success($info);
+    }
+    /**
+     * 修改资讯
+     * @param array $data
+     * @return array
+     */
+    public function upInfo(array $data): array
+    {
+        $id = $data['art_id'];
+        $content = $data['content'];
+        unset($data['art_id']);
+        //去掉此元素
+        unset($data['content']);
+        //去掉此元素
+        $info =  Article::where('id',$id)->first();  
+        if($info==null){
+            return Result::error('请输入正确的文章id！');
+        }
+        if($info['state']==1){
+            return Result::error('此文章已导入 ，不可编辑！');
+        }else{
+            Db::beginTransaction();
+            try{
+                $info =  Article::where('id',$id)->update($data);  
+                $art_data = ArticleData::where('article_id',$id)->update(['content'=>$content]);
+
+                Db::commit();
+            } catch(\Throwable $ex){
+                Db::rollBack();
+                var_dump($ex->getMessage());
+                return Result::error("修改失败",0);
+            }
+            $data = [
+                'info' => $info,
+                'art_data' => $art_data
+            ];
+            return  Result::success($data);
+        }
+        
+    }
+    /**
+    * 删除资讯
+    * @param array $data
+    * @return array
+    */
+    public function delInfo(array $data): array
+    {
+        $id = $data['art_id'];
+        $info = Article::where('id',$id)->first();
+        if($info==null){
+            return Result::error('请输入正确的文章id！');
+        }
+        if($info['state']==1){
+            return Result::error('此文章已导入，不可删除！'); 
+        }else{
+            Db::beginTransaction();
+            try{
+                $delinfo = Article::where('id',$id)->delete();
+                $deldata = ArticleData::where('article_id',$id)->delete();
+                Db::commit();
+            } catch(\Throwable $ex){
+                Db::rollBack();
+                var_dump($ex->getMessage());
+                return Result::error("删除失败",0);
+            }
+        }
+        $data = [
+            'delinfo' => $delinfo,
+            'deldata' => $deldata
+        ];
+        return  Result::success($data);
+    }
+    /**
+    * 关联导航池
+    * @param array $data
+    * @return array
+    */
+    public function addCatid(array $data): array
+    {
+        $id = $data['rule_id'];
+        $art = Article::where('rule_id',$id)->select('id')->count();
+        if($art==0){
+            return Result::error('还未采集，请采集');
+        }else{
+            $info = Article::where('rule_id',$id)->where('state',0)->select('id')->get();
+        if(empty($info->toArray())){
+            return Result::error('所有文章都已导入，不可修改关联的导航池！'); 
+        }else{
+            //查找此规则任务下的文章是否已经有导入的文章
+            $article = Article::where('rule_id',$id)->where('state',1)->select('id')->get();
+            if(!empty($article->toArray())){
+                //若有已导入的文章则直接复制之前已导入的导航池
+                $catid = Article::whereIn('id',$article)->select('catid')->first();
+                //若未导入的文章已经复制之前的导航，则无需修改
+                $art_catid = Article::whereIn('id',$info)->whereNull('catid')->count();
+                if($art_catid>0){
+                    $result = Article::whereIn('id',$info)->update(['catid'=>$catid['catid']]);
+                }else{
+                    $result = ['已全部关联导航，无需再次关联！'];
+                }
+            }else{
+                //若不存在已导入的文章则判断是否存在导航id
+                if(isset($data['catid'])){
+                    //若存在直接使用此导航id
+                    $result = Article::whereIn('id',$info)->update(['catid'=>$data['catid']]);
+                    
+                }else{
+                    //若不存在则返回所有导航栏目
+                    $result = Category::select('id','name')->get();
+                }
+            }
+              
+        }
+        }
+        //查找此规则任务下的文章是否都已经导入
+        
+        return  Result::success($result);
+    }
+    /**
+    * 导入文章
+    * @param array $data
+    * @return array
+    */
+    public function addArt(array $data): array
+    {
+        // var_dump("======@@@====");
+        $where = [
+            'rule_id' => $data['rule_id'],
+            'state' => 0
+        ];
+        //获取某个规则任务下的已采集未导入的文章及文章详情
+        $arts_id = Article::where($where)->wherenotNull('catid')->orderBy('id')->select('id')->get();
+        $arts = Article::where($where)->wherenotNull('catid')->select('title','catid','level','introduce','keyword','author','copyfrom','fromurl','hits','islink','imgurl','admin_user_id','is_original')->orderBy('id')->get()->toArray();       
+        $arts_data = ArticleData::whereIn('article_id',$arts_id)->select('content')->orderBy('article_id')->get()->toArray();
+        // var_dump($article_data);
+        $data = [
+            'articles' => $arts,
+            'art_content' => $arts_data
+        ];
+        Db::beginTransaction();
+        try{
+           
+            $oldart = OldArticle::insert($arts);
+            $oldart_data = OldArticleData::insert($arts_data);
+            $upstate_art = Article::where($where)->wherenotNull('catid')->update(['state' => 1]);
+            Db::commit();
+            
+        } catch(\Throwable $ex){
+            Db::rollBack();
+            var_dump($ex->getMessage());
+            return Result::error($ex->getMessage(),0);
+        }
+        return Result::success($data);
+    }
 }

app/JsonRpc/CollectorServiceInterface.php

@@ -23,12 +23,63 @@ interface CollectorServiceInterface
      *  @return array
     */
     public function delWeb(array $data):array;
+    /**
+     * @param array $data
+     *  @return array
+    */
+    public function addRule(array $data):array;
+    /**
+     * @param array $data
+     *  @return array
+    */
+    public function getRule(array $data):array;
+     /**
+     * @param array $data
+     *  @return array
+    */
+    public function getOneRule(array $data):array;
+    /**
+     * @param array $data
+     *  @return array
+    */
+    public function upRule(array $data):array;
+
      /**
      * @param array $data
      * @return array
      */
     public function sendCrawler(array $data): array;
 
+    /**
+     * @param array $data
+     * @return array
+     */
+    public function getInfo(array $data): array;
+    /**
+     * @param array $data
+     * @return array
+     */
+    public function getOneInfo(array $data): array;
+     /**
+     * @param array $data
+     * @return array
+     */
+    public function upInfo(array $data): array;
+     /**
+     * @param array $data
+     * @return array
+     */
+    public function delInfo(array $data): array;
+     /**
+     * @param array $data
+     * @return array
+     */
+    public function addCatid(array $data): array;
+     /**
+     * @param array $data
+     * @return array
+     */
+    public function addArt(array $data): array;
 }
 
 

app/Model/Article.php

@@ -24,4 +24,12 @@ class Article extends Model
      * The attributes that should be cast to native types.
      */
     protected array $casts = [];
+    public function rule()
+    {
+        return $this->belongsTo(Rule::class,'rule_id','id');
+    }
+    public function category()
+    {
+        return $this->belongsTo(OldModel\Category::class, 'catid','id');
+    }
 }

app/Model/OldModel/Article.php

@@ -19,7 +19,7 @@ class Article extends Model
     /**
      * The attributes that are mass assignable.
      */
-    protected array $fillable = [];
+    protected array $fillable = ['title','catid','level','introduce','keyword','author','copyfrom','fromurl','hits','islink','imgurl','source','admin_user_id','created_at','updated_at','is_original'];
 
     /**
      * The attributes that should be cast to native types.

app/Model/OldModel/ArticleData.php

@@ -0,0 +1,28 @@
+<?php
+
+declare(strict_types=1);
+
+namespace App\Model\OldModel;
+
+use Hyperf\DbConnection\Model\Model;
+
+/**
+ */
+class ArticleData extends Model
+{
+    /**
+     * The table associated with the model.
+     */
+    protected ?string $table = 'article_data';
+    protected ?string $connection = 'secondary';
+
+    /**
+     * The attributes that are mass assignable.
+     */
+    protected array $fillable = ['content'];
+
+    /**
+     * The attributes that should be cast to native types.
+     */
+    protected array $casts = [];
+}

app/Model/OldModel/Category.php

@@ -0,0 +1,32 @@
+<?php
+
+declare(strict_types=1);
+
+namespace App\Model\OldModel;
+
+use Hyperf\DbConnection\Model\Model;
+
+/**
+ */
+class Category extends Model
+{
+    /**
+     * The table associated with the model.
+     */
+    protected ?string $table = 'category';
+    protected ?string $connection = 'secondary';
+
+    /**
+     * The attributes that are mass assignable.
+     */
+    protected array $fillable = [];
+
+    /**
+     * The attributes that should be cast to native types.
+     */
+    protected array $casts = [];
+    public function info()
+    {
+        return $this->hasMany(Article::class, 'id','catid');
+    }
+}

app/Model/Rule.php

@@ -24,4 +24,11 @@ class Rule extends Model
      * The attributes that should be cast to native types.
      */
     protected array $casts = [];
+<<<<<<< HEAD
+=======
+    public function arts()
+    {
+        return $this->hasMany(Article::class,'rule_id','id');
+    }
+>>>>>>> collector_1106_fr
 }

runtime/container/proxy/App_Controller_AbstractController.proxy.php

@@ -1,32 +0,0 @@
-<?php
-
-declare (strict_types=1);
-/**
- * This file is part of Hyperf.
- *
- * @link     https://www.hyperf.io
- * @document https://hyperf.wiki
- * @contact  group@hyperf.io
- * @license  https://github.com/hyperf/hyperf/blob/master/LICENSE
- */
-namespace App\Controller;
-
-use Hyperf\Di\Annotation\Inject;
-use Hyperf\HttpServer\Contract\RequestInterface;
-use Hyperf\HttpServer\Contract\ResponseInterface;
-use Psr\Container\ContainerInterface;
-abstract class AbstractController
-{
-    use \Hyperf\Di\Aop\ProxyTrait;
-    use \Hyperf\Di\Aop\PropertyHandlerTrait;
-    function __construct()
-    {
-        $this->__handlePropertyHandler(__CLASS__);
-    }
-    #[Inject]
-    protected ContainerInterface $container;
-    #[Inject]
-    protected RequestInterface $request;
-    #[Inject]
-    protected ResponseInterface $response;
-}