imported data error

crontab/crawl/brand_operator.php

@@ -13,6 +13,7 @@ class brand_operator
     private $mod_brand;
     private $tm_goods;
     private $mod_goods_common;
+    private $mod_goods;
 
     const brand_pid = 20000;
 
@@ -23,6 +24,7 @@ class brand_operator
         $this->cls_tree = new class_tree();
         $this->mod_brand = Model('brand');
         $this->mod_goods_common = Model('goods_common');
+        $this->mod_goods = Model('goods');
     }
 
     private function brand_exist($name)
@@ -36,7 +38,7 @@ class brand_operator
         }
     }
 
-    private function create()
+    public function create()
     {
         $brands = $this->tm_prop_values->field('id,cid,name')->where(array('pid' => self::brand_pid))->group('name')->select();
 
@@ -80,10 +82,10 @@ class brand_operator
         return false;
     }
 
-    public function proc()
-    {
-        $this->create();
 
+
+    public function update_brand()
+    {
         $results = $this->tm_goods->field('body,title,num_iid')->limit(false)->select();
         foreach ($results as $result)
         {
@@ -99,6 +101,8 @@ class brand_operator
                 $ret = $this->mod_goods_common->where(array('num_iid' => $num_iid))->update(array('brand_id' => $brand_id,'brand_name' => $brand_name));
                 if($ret == false) {
                     Log::record("update goods_common brand error: num_iid={$num_iid},brand_id={$brand_id},brand_name={$brand_name}");
+                } else {
+                    $ret = $this->mod_goods->where(array('num_iid' => $num_iid))->update(array('brand_id' => $brand_id));
                 }
             } else {
                 Log::record("update goods_common brand error: cannot find brand_id num_iid={$num_iid}");
@@ -115,7 +119,8 @@ class brand_operator
         $spec_vals = preg_split("/[;]+/", $propnames);
 
         $ret = array();
-        foreach ($spec_vals as $sv) {
+        foreach ($spec_vals as $sv)
+        {
             $data = preg_split("/[:]+/", $sv);
             if (!empty($data)) {
                 $tmpid = $data[0];

crontab/crawl/fetch_config.php

@@ -0,0 +1,16 @@
+<?php
+
+/**
+ * Created by PhpStorm.
+ * User: stanley-king
+ * Date: 16/1/18
+ * Time: 下午8:00
+ */
+
+class fetch_config
+{
+    const store_name = '熊猫美妆';
+    const store_id = 3;
+    //const tmp_file = '/private/var/tmp/phpxxkkdl';
+    const tmp_file = '/tmp/phpxxkkdl';
+}

crontab/crawl/fetch_goods.php

@@ -3,9 +3,6 @@
 define('InShopNC',true);
 defined('InShopNC') or exit('Access Invalid!');
 
-echo "5\r\n";
-
-
 require_once (BASE_DATA_PATH . '/api/taobao/TopSdk.php');
 require_once (BASE_DATA_PATH . '/api/taobao/top/TopClient.php');
 require_once (BASE_DATA_PATH . '/api/taobao/top/ResultSet.php');
@@ -16,7 +13,8 @@ require_once (BASE_DATA_PATH . '/api/taobao/top/ApplicationVar.php');
 require_once (BASE_DATA_PATH . '/api/taobao/top/HttpdnsGetRequest.php');
 require_once (BASE_DATA_PATH . '/api/taobao/top/request/ItemSellerGetRequest.php');
 require_once (BASE_DATA_PATH . '/api/taobao/top/request/ItemsOnsaleGetRequest.php');
-echo "6\r\n";
+
+require_once (BASE_CRONTAB_PATH . '/crawl/fetch_config.php');
 require_once (BASE_CRONTAB_PATH . '/crawl/product_importer.php');
 require_once (BASE_CRONTAB_PATH . '/crawl/category.php');
 require_once (BASE_CRONTAB_PATH . '/crawl/tmrequest.php');
@@ -31,10 +29,8 @@ require_once (BASE_CRONTAB_PATH . '/crawl/tm_pagereq.php');
 require_once(BASE_CRONTAB_PATH . '/crawl/brand_operator.php');
 
 
-echo "7\r\n";
 //require_once (BASE_CORE_PATH . '/framework/libraries/uploadfile.php');
 //require_once (BASE_CRONTAB_PATH . '/crawl/skumatcher.php');
-echo "8\r\n";
 
 
 date_default_timezone_set('Asia/Shanghai');
@@ -54,22 +50,24 @@ function not_null($str)
 }
 
 
-
-
 //$tmreq = new tmrequest();
 //$tmreq->proc();
-//
+
+$treater = new brand_operator();
+
+
 //$cat = new category();
 //$cat->proc();
 //
-$treater = new pretreat_tmdata();
-$treater->proc(false);
-//
+//$treater = new pretreat_tmdata();
+//$treater->proc(false);
+
 //$spec = new spec_attr_parser();
 //$spec->proc();
-//
-//$importer = new product_importer();
-//$importer->proc_shop();
 
-$treater = new brand_operator();
-$treater->proc();
+$importer = new product_importer();
+//$importer->proc_shop();
+$importer->proc();
+//
+//
+$treater->update_brand();

crontab/crawl/product_importer.php

@@ -8,6 +8,7 @@
 
 require_once(BASE_DATA_PATH . '/model/goods.model.php');
 
+
 class product_importer
 {
     private $tm_goods;
@@ -31,9 +32,6 @@ class product_importer
     private $cls_tree;
     private $down_path;
 
-    const store_name = "熊猫美妆";
-    const store_id = 6;
-
     public function __construct()
     {
         $this->tm_goods = Model('tm_goods');
@@ -285,8 +283,9 @@ class product_importer
 
         $general['num_iid'] = $num_iid;
         $general['tm_shop'] = $shop_id;
-        $general['store_id'] = self::store_id;
-        $general['store_name'] = self::store_name;
+        $general['store_id'] =  fetch_config::store_id;
+        $general['store_name'] = fetch_config::store_name;
+
         $general['goods_name'] = not_null($item['title']);
         $general['goods_mobile_name'] = not_null($item['title']);
         $general['goods_jingle'] = not_null($item['sell_point']);
@@ -318,17 +317,52 @@ class product_importer
 
     private function upload_imges(&$imges,$storeid)
     {
-        foreach($imges as &$img) {
+        foreach($imges as &$img)
+        {
+            static $pngext = array('png','jpg','jpeg');
+
             $url = $img['url'];
             $info = pathinfo($url);
             $src_name = $this->down_path . '/' . md5($url) . '.' . $info['extension'];
-            if(!file_exists($src_name) || filesize($src_name) == 0) continue;
+
+            if(empty($info['extension'])) continue;
+
+            $err = true;
+            if(file_exists($src_name))
+            {
+                $fh = fopen($src_name, "rb");
+                $head = fread($fh, 8);
+                fclose($fh);
+
+                $err = false;
+                if(empty($head)) {
+                    unlink($src_name);
+                    Log::record("{$src_name} file is empty.",Log::ERR);
+                    $err = true;
+                }
+
+                if(in_array($info['extension'],$pngext))
+                {
+                    $arr = unpack("C4", $head);
+                    if($arr[1] == 137 &&  $arr[2] == 80) {
+                        Log::record("{$src_name} is a png file.",Log::ERR);
+                        $err = true;
+                    }
+                }
+            }
+            if($err) {
+                Log::record("{$src_name} file is error.",Log::ERR);
+                continue;
+            }
+
             $upctl = new upload_control();
             $upctl->set_store($storeid);
             $up_name = $upctl->upload_img($src_name);
             $img['upname'] = $up_name;
             $img['file'] = $src_name;
         }
+
+        return true;
     }
 
     private function add_attr($gd_id,$gdcommon_id,$gc_id,$type_id,$goods_attr)
@@ -371,8 +405,6 @@ class product_importer
         return 0;
     }
 
-
-
     private function import_item($item,$desc,$num_iid,$shop_id,&$nsku)
     {
         $cid = $item['cid'];
@@ -409,11 +441,11 @@ class product_importer
 
             $prop_imgs = $item['prop_imgs']['prop_img'];
             if(!empty($prop_imgs)) {
-                $this->upload_imges($prop_imgs,self::store_id);
+                $this->upload_imges($prop_imgs,fetch_config::store_id);
             }
 
             $imgs = $this->reset_imgs($item['item_imgs']['item_img']);
-            $this->upload_imges($imgs,self::store_id);
+            $this->upload_imges($imgs,fetch_config::store_id);
 
             if(empty($prop_imgs))
             {
@@ -434,7 +466,7 @@ class product_importer
                     }
 
                     $this->mod_goods_images->insert(array('goods_commonid' => $common_id,
-                        'store_id' => self::store_id,'goods_image' => $upname,'goods_image_sort' => $pos,'is_default' => $default,'filesig' => $file_sig));
+                        'store_id' => fetch_config::store_id,'goods_image' => $upname,'goods_image_sort' => $pos,'is_default' => $default,'filesig' => $file_sig));
                 }
             }
             else
@@ -453,7 +485,7 @@ class product_importer
 
                     $color_id = $this->goods_colorid($cid,$pimg['properties']);
                     $this->mod_goods_images->insert(array('goods_commonid' => $common_id,'color_id' => $color_id,
-                        'store_id' => self::store_id,'goods_image' => $pupname,'goods_image_sort' => $pos,'is_default' => $default,'filesig' => $file_sig));
+                        'store_id' => fetch_config::store_id,'goods_image' => $pupname,'goods_image_sort' => $pos,'is_default' => $default,'filesig' => $file_sig));
                     $main_pic = $pupname;
 
                     $default = 0;
@@ -467,7 +499,7 @@ class product_importer
                             $file_sig = md5_file($img['file']);
                         }
                         $this->mod_goods_images->insert(array('goods_commonid' => $common_id,'color_id' => $color_id,
-                            'store_id' => self::store_id,'goods_image' => $upname,'goods_image_sort' => $pos + 1,'is_default' => $default,'filesig' => $file_sig));
+                            'store_id' => fetch_config::store_id,'goods_image' => $upname,'goods_image_sort' => $pos + 1,'is_default' => $default,'filesig' => $file_sig));
                     }
                 }
             }
@@ -523,19 +555,31 @@ class product_importer
         return $common_id;
     }
 
-//    public function proc()
-//    {
-//        $bodys = $this->tm_goods->field('num_iid,shop_id,body,description')->where(array('imported' => 0))->limit(false)->order('num_iid')->select();
-//
-//        $items = $this->get_tmgoods($bodys);
-//        foreach($items as $item) {
-//            print "handle num_iid=" . $item['num_iid'] . " start.\r\n";
-//            $this->import_item($item['body'],$item['desc'],$item['num_iid'],$item['shop_id']);
-//
-//            $this->tm_goods->where(array('num_iid' => $item['num_iid']))->update(array('imported' => 1));
-//            print "handle num_iid=" . $item['num_iid'] . " end. \r\n";
-//        }
-//    }
+    public function proc()
+    {
+        $condition = array('num_iid' => 13898341378);
+        //$condition = array('imported' => 0);
+        $bodys = $this->tm_goods->field('num_iid,shop_id,body,description')->where($condition)->limit(false)->order('num_iid')->select();
+
+        $items = $this->get_tmgoods($bodys);
+        foreach($items as $item) {
+            print "handle num_iid=" . $item['num_iid'] . " start.\r\n";
+            $this->import_item($item['body'],$item['desc'],$item['num_iid'],$item['shop_id']);
+
+            $this->tm_goods->where(array('num_iid' => $item['num_iid']))->update(array('imported' => 1));
+            print "handle num_iid=" . $item['num_iid'] . " end. \r\n";
+        }
+    }
+
+    private function goods_exist($num_iid)
+    {
+        $items = $this->mod_goods->where(array('num_iid' => $num_iid))->select();
+        if(!empty($items) && count($items) > 0) {
+            return true;
+        } else {
+            return false;
+        }
+    }
 
     public function proc_shop()
     {
@@ -549,7 +593,8 @@ class product_importer
             Log::endl(Log::DEBUG);
             Log::record("begin shop_id={$shopid} spu count= {$count}",Log::DEBUG);
 
-            $bodys = $this->tm_goods->field('num_iid,cid,shop_id,body,description,picnum')->where(array('imported' => 0,'shop_id' => $shopid))->limit(false)->order('num_iid')->select();
+            $con = array('imported' => 0,'shop_id' => $shopid);
+            $bodys = $this->tm_goods->field('num_iid,cid,shop_id,body,description,picnum')->where($con)->limit(false)->order('num_iid')->select();
 
             $items = $this->get_tmgoods($bodys);
             foreach($items as $item)
@@ -560,17 +605,19 @@ class product_importer
                 Log::endl(Log::DEBUG);
                 Log::record("handle num_iid={$num_iid} start.\r\n",Log::DEBUG);
 
-                echo "begin shop_id={$shopid} spu count= {$count} \r\n";
+                if($this->goods_exist($num_iid)) {
+                    Log::record("handle num_iid={$item['num_iid']} has existed.",Log::DEBUG);
+                    continue;
+                }
 
                 $cid = $item['cid'];
                 if(is_excids($cid)) {
                     --$count;
+                    Log::record("handle num_iid={$item['num_iid']} cid is exclude.",Log::DEBUG);
                     continue;
                 }
 
-                echo __FUNCTION__ . " 1 \r\n";
                 $commonid =$this->import_item($item['body'],$item['desc'],$num_iid,$item['shop_id'],$nsku);
-                echo __FUNCTION__ . " 2 \r\n";
                 if($commonid)
                 {
                     if($this->check($num_iid,$nsku,$picnum,$commonid) == false) {
@@ -578,6 +625,9 @@ class product_importer
                         $this->mod_goods_common->where(array('goods_commonid' => $commonid))->delete();
                         $this->mod_goods_images->where(array('goods_commonid' => $commonid))->delete();
                         $this->mod_goods->where(array('goods_commonid' => $commonid))->delete();
+                        $this->mod_goods->where(array('num_iid' => $num_iid))->delete();
+                        $this->mod_goods_common->where(array('num_iid' => $num_iid))->delete();
+
                         Log::record("shopid = {$shopid} and num_iid = {$num_iid} nsku = {$nsku} cannot be imported.",Log::ERR);
                     } else {
                         $this->tm_goods->where(array('num_iid' => $item['num_iid']))->update(array('imported' => 1,'commonid' => $commonid));
@@ -590,8 +640,6 @@ class product_importer
                 }
 
                 Log::record("handle num_iid={$item['num_iid']} end.",Log::DEBUG);
-
-                echo "handle num_iid={$item['num_iid']} end.";
             }
 
             Log::record("end shop_id = {$shopid} Error= {$count}",Log::DEBUG);

crontab/crawl/product_item.php

@@ -6,6 +6,9 @@
  * Date: 15/11/3
  * Time: 下午8:29
  */
+
+//server
+
 class product_item
 {
     private $mod_spec_ex;
@@ -17,7 +20,26 @@ class product_item
 
     public function proc_item()
     {
+    }
+
+    protected function  xxx()
+    {
+
+    }
+}
 
+class B extends product_item
+{
+
+    public function __construct()
+    {
+        parent::__construct();
     }
 
-}
+}
+
+//client
+
+$x = new product_item;
+$x->proc_item();
+

crontab/crawl/spec_attr.php

@@ -25,9 +25,6 @@ class spec_attr
     private $mod_attribute_value;
     private $tm_tvid_attrvid;
 
-    const store_name = "美宝莲";
-    const store_id = 1;
-
     public function __construct()
     {
         $this->tm_spec = Model('tm_spec');
@@ -123,7 +120,7 @@ class spec_attr
                     $spec_val = array();
                     $spec_val['sp_id'] = $pid;
                     $spec_val['sp_value_name'] = $item['name'];
-                    $spec_val['store_id'] = 1;
+                    $spec_val['store_id'] = fetch_config::store_id;
                     $spec_val['sp_value_color'] = '#ddd9c3';
                     $spec_val['gc_id'] = $cid;
                     $spec_val['sp_value_sort'] = 0;
@@ -210,7 +207,8 @@ class spec_attr
                 $attr_multi = $multi ? 1 : 0;
 
                 $attr_value = $this->gen_attr_value($cid,$pid);
-                $attrid = $this->mod_attribute->table('attribute')->insert(array( "attr_name" => $name,"type_id" => $cid,'attr_value' => $attr_value, "attr_multi" => $attr_multi,"attr_show" => 1));
+                $attrid = $this->mod_attribute->table('attribute')->insert(array( "attr_name" => $name,"type_id" => $cid,
+                    'attr_value' => $attr_value, "attr_multi" => $attr_multi,"attr_show" => 1));
 
                 if($attrid)
                 { // 建立shopnc attribute id 与 天猫数据 pid ＋ cid 的对应关系

crontab/crawl/spec_attr_parser.php

@@ -67,28 +67,49 @@ class spec_attr_parser
     private function add_spec()
     {
         $datas = $this->tm_spec->group('props')->order('props')->limit(false)->select(); //获取所有独立的规格
-        foreach ($datas as $item) {
+        foreach ($datas as $item)
+        {
             $cid = $item['cid'];
-            if(is_excids($cid)) continue;
+            if(is_excids($cid)) {
+                Log::record("num_iid = {$item['num_iid']} has been excluded.");
+                continue;
+            }
 
             $pid = $item['props'];
+            if(empty($pid)) {
+                Log::record("num_iid = {$item['num_iid']} props is null.");
+                continue;
+            }
+
             $spec_name = $this->get_tm_props_name($cid, $pid);
-            $sp_id = $this->mod_spec->insert(array('sp_id' => $pid, 'sp_name' => $spec_name));
-            if($sp_id != $pid) {
-                Log::record(__FUNCTION__ . ':' . __LINE__ . "  cannt find prop values from {$cid} {$pid}",Log::ERR);
+            if(empty($spec_name)) {
+                Log::record("num_iid = {$item['num_iid']} spec_nme is empty.");
+                continue;
             }
+
+            $this->mod_spec->insert(array('sp_id' => $pid, 'sp_name' => $spec_name));
         }
     }
 
     private function add_type()
     {
+        Log::record(__FILE__ ."   " . __FUNCTION__,Log::DEBUG);
         //通过分类直接建立类型
         $items = $this->tm_spec->field('cid')->group('cid')->limit(false)->select();
-        foreach($items as $item) {
+        foreach($items as $item)
+        {
             $cid = $item['cid']; // $cid => $typeid 以分类建立类型
-            if(is_excids($cid)) continue;
+            if(is_excids($cid)) {
+                Log::record("num_iid = {$item['num_iid']} has been excluded.");
+                continue;
+            }
 
             $cnmae = $this->get_class_name($cid);
+            if(empty($cnmae)) {
+                Log::record("num_iid = {$item['num_iid']} cannot find cid = {$cid}.");
+                continue;
+            }
+
             $this->mod_type->insert(array("type_id" => $cid, "type_name" => $cnmae));
             $this->mod_good_class->where(array("gc_id" => $cid))->update(array("type_id" => $cid, "type_name" => $cnmae));
         }
@@ -103,16 +124,22 @@ class spec_attr_parser
     private function add_type_spec()
     {
         //按照分类建立类型
-        $items = $this->tm_spec->field('cid,props')->group('cid,props')->limit(false)->select();
+        $items = $this->tm_spec->field('cid,props,num_iid')->group('cid,props')->limit(false)->select();
         foreach($items as $item)
         {
             $cid = $item['cid']; // $cid => $typeid 以分类建立类型
-            if(is_excids($cid)) continue;
-
+            if(is_excids($cid)) {
+                Log::record("num_iid = {$item['num_iid']} has been excluded.");
+                continue;
+            }
             $pid = $item['props'];
 
-            if($this->type_spec_exist($cid,$pid)) continue;
+            if(empty($pid)) {
+                Log::record("num_iid = {$item['num_iid']} pid is empty.");
+                continue;
+            }
 
+            if($this->type_spec_exist($cid,$pid)) continue;
             $this->mod_type_spec->insert(array("type_id" => $cid,"sp_id" => $pid));
         }
     }
@@ -122,10 +149,11 @@ class spec_attr_parser
         $bodys = $this->tm_goods->field('num_iid,body')->where(array('imported' => 0))->limit(false)->order('num_iid')->select();
         $result = array();
 
-        foreach($bodys as $body) {
+        foreach($bodys as $body)
+        {
             $item = json_decode($body['body'],true);
             if(empty($item)) {
-                Log::record('Cannot decode num_iid='.$body['num_iid'].' goods.');
+                Log::record("Cannot decode num_iid={$body['num_iid']} goods.");
                 continue;
             }
             $product = array();
@@ -185,7 +213,7 @@ class spec_attr_parser
             $spec_val = array();
             $spec_val['sp_id'] = $spvals['pid'];
             $spec_val['sp_value_name'] = $spvals['vname'];
-            $spec_val['store_id'] = 1;
+            $spec_val['store_id'] = fetch_config::store_id;
             $spec_val['sp_value_color'] = '#ddd9c3';
             $spec_val['gc_id'] = $cid;
             $spec_val['sp_value_sort'] = 0;
@@ -213,11 +241,20 @@ class spec_attr_parser
         foreach ($tmgoods as $goods)
         {
             $cid = $goods['body']['cid'];
-            if(is_excids($cid)) continue;
+            if(is_excids($cid)) {
+                continue;
+            }
 
             $skus = $goods['body']['skus'];
-            if(empty($skus)) continue;
+            if(empty($skus)) {
+                continue;
+            }
+
             $skus = $skus['sku'];
+            if(empty($skus)) {
+                continue;
+            }
+
             foreach($skus as $sku) {
                 $prpos_name = $this->parse_properties_name($sku['properties_name']);
                 $this->add_spval($prpos_name,$cid);

crontab/crawl/transport_data.php

@@ -157,7 +157,7 @@ class ModelOperator
                 $spec_val['sp_id'] = $pid;
                 //$spec_val['sp_value_id'] = $item['vid']; //此处不能赋值，因为sp_value_id 是自索引的
                 $spec_val['sp_value_name'] = $item['name'];
-                $spec_val['store_id'] = 1;
+                $spec_val['store_id'] = fetch_config::store_id;
                 $spec_val['sp_value_color'] = '#ddd9c3';
                 $spec_val['gc_id'] = $cid;
                 $spec_val['sp_value_sort'] = 0;

crontab/crawl/upload_control.php

@@ -27,26 +27,25 @@ class upload_control
     }
 
     const POST_NAME = 'goods_image';
-    //const tmp_file = '/private/var/tmp/phpxxkkdl';
-    const tmp_file = '/tmp/phpxxkkdl';
 
     private function fill_files($filename)
     {
-        if(file_exists(self::tmp_file)) {
-              unlink(self::tmp_file);
+        if(file_exists(fetch_config::tmp_file)) {
+              unlink(fetch_config::tmp_file);
         }
-        copy($filename,self::tmp_file);
+        copy($filename,fetch_config::tmp_file);
         if(empty($_FILES['goods_image'])) {
             $_FILES['goods_image'] = array();
-            $post_info = &$_FILES['goods_image'];
         }
+
+        $post_info = &$_FILES['goods_image'];
         $path_info = pathinfo($filename);
         $post_info['path'] = $filename;
         $post_info['name'] = $path_info['basename'];
         $post_info['type'] = 'image/'.$path_info['extension'];
         $post_info['error'] = 0;
         $post_info['size'] = filesize($filename);
-        $post_info['tmp_name'] = self::tmp_file;
+        $post_info['tmp_name'] = fetch_config::tmp_file;
     }
 
     public function upload_img($file)