add check goods

core/framework/libraries/log.php

@@ -51,6 +51,22 @@ class Log
         }
     }
 
+    public static function endl($lev = self::ERR)
+    {
+        $content = '\r\n';
+
+        if($lev == self::SQL && self::open_sql) {
+            $log_file = BASE_DATA_PATH.'/log/'.date('Ymd',TIMESTAMP).'-sql.log';
+            file_put_contents($log_file,$content, FILE_APPEND);
+            return;
+        }
+
+        if($lev >= self::cur_level) {
+            $log_file = BASE_DATA_PATH . '/log/' . date('Ymd', TIMESTAMP) . '.log';
+            file_put_contents($log_file, $content, FILE_APPEND);
+        }
+    }
+
     private static function get_level($lev)
     {
         if($lev == self::INFO) return 'INFO';

crontab/crawl/category.php

@@ -74,7 +74,6 @@ class category
         $items = $this->goods_class->field('*')->where(array('gc_id' => $cid))->limit(false)->select();
 
         if(empty($items)) {
-            Log::record("gc_id = {$cid} cannot get info from goods_class.",Log::ERR);
             return NULL;
         } else {
             return $items[0];
@@ -86,7 +85,6 @@ class category
         $items = $this->tm_cats->field('*')->where(array('cid' => $cid))->limit(false)->select();
 
         if(empty($items)) {
-            Log::record("cid = {$cid} cannot get info from tm_cats",Log::ERR);
             return NULL;
         } else {
             return $items[0];

crontab/crawl/fetch_goods.php

@@ -45,12 +45,12 @@ function not_null($str)
 ////
 //$cat = new category();
 //$cat->proc();
-
+//
 //$treater = new pretreat_tmdata();
-//$treater->proc(true);
+//$treater->proc(false);
 //
 //$spec = new spec_attr_parser();
 //$spec->proc();
 
-//$importer = new product_importer();
-//$importer->proc_shop();
+$importer = new product_importer();
+$importer->proc_shop();

crontab/crawl/pretreat_tmdata.php

@@ -28,6 +28,7 @@ class pretreat_tmdata
         $this->tm_spec->where(array('num_iid > 0'))->delete();
         $results = $this->tm_goods->field('body,title,num_iid')->where(array('imported' => 0))->limit(false)->select();
         foreach ($results as $result) {
+            $num_iid = $result['num_iid'];
             $body = $result['body'];
             $response = json_decode($body, true);
             $item = $response['item_seller_get_response']['item'];
@@ -36,7 +37,11 @@ class pretreat_tmdata
             if(is_excids($cid)) continue;
 
             $this->parse_tm_product($item);
-            $this->parse_tm_pic($item);
+
+            //下载图片，并确认图片数量，用于验证后面商品是否成功导入
+            $picnum = 0;
+            $this->parse_tm_pic($item,$picnum);
+            $this->tm_goods->where(array('num_iid' => $num_iid))->update(array('picnum' => $picnum));
         }
         if($create_brand == true) {
             $this->create_brand();
@@ -134,9 +139,15 @@ class pretreat_tmdata
             $pic_url = $info['dirname'] . '/' . urlencode($info['basename']);
             exec("wget -O $path $pic_url");
         }
+
+        if(file_exists($path)) {
+            return true;
+        } else {
+            return false;
+        }
     }
 
-    private function parse_tm_pic($item)
+    private function parse_tm_pic($item,&$npics)
     {
         $pics = array();
         $main_pic = $item['pic_url'];
@@ -161,8 +172,22 @@ class pretreat_tmdata
             }
         }
 
-        foreach($pics as $pic) {
-            $this->download($pic);
+        $arpic = array();
+        foreach($pics as $pic)
+        {
+            $name = md5($pic);
+            if(array_key_exists($name,$arpic) == false)
+            {
+                if($this->download($pic)) {
+                    $arpic[$name] = 1;
+                }
+            } else {
+                //$arpic[$name] += 1;
+            }
+        }
+
+        foreach($arpic as $key => $val) {
+            $npics += $val;
         }
     }
 

crontab/crawl/product_importer.php

@@ -72,6 +72,7 @@ class product_importer
             $product['desc'] = $body['description'];
             $product['num_iid'] = $body['num_iid'];
             $product['shop_id'] = $body['shop_id'];
+            $product['picnum'] = $body['picnum'];
 
             array_push($result,$product);
         }
@@ -326,6 +327,7 @@ class product_importer
             $upctl->set_store($storeid);
             $up_name = $upctl->upload_img($src_name);
             $img['upname'] = $up_name;
+            $img['file'] = $src_name;
         }
     }
 
@@ -426,8 +428,13 @@ class product_importer
                         $default = 0;
                     }
 
+                    $file_sig = '';
+                    if(isset($img['file'])) {
+                        $file_sig = md5_file($img['file']);
+                    }
+
                     $this->mod_goods_images->insert(array('goods_commonid' => $common_id,
-                        'store_id' => self::store_id,'goods_image' => $upname,'goods_image_sort' => $pos,'is_default' => $default));
+                        'store_id' => self::store_id,'goods_image' => $upname,'goods_image_sort' => $pos,'is_default' => $default,'filesig' => $file_sig));
                 }
             }
             else
@@ -437,9 +444,16 @@ class product_importer
                     $default = 1;
                     $pos = 0;
                     $pupname = $pimg['upname'];
+                    if(empty($pupname)) continue;
+
+                    $file_sig = '';
+                    if(isset($img['file'])) {
+                        $file_sig = md5_file($img['file']);
+                    }
+
                     $color_id = $this->goods_colorid($cid,$pimg['properties']);
                     $this->mod_goods_images->insert(array('goods_commonid' => $common_id,'color_id' => $color_id,
-                        'store_id' => self::store_id,'goods_image' => $pupname,'goods_image_sort' => $pos,'is_default' => $default));
+                        'store_id' => self::store_id,'goods_image' => $pupname,'goods_image_sort' => $pos,'is_default' => $default,'filesig' => $file_sig));
                     $main_pic = $pupname;
 
                     $default = 0;
@@ -447,9 +461,13 @@ class product_importer
                     {
                         $upname = $img['upname'];
                         if(empty($upname)) continue;
-                        //$color_id = $this->goods_colorid($cid);
+
+                        $file_sig = '';
+                        if(isset($img['file'])) {
+                            $file_sig = md5_file($img['file']);
+                        }
                         $this->mod_goods_images->insert(array('goods_commonid' => $common_id,'color_id' => $color_id,
-                            'store_id' => self::store_id,'goods_image' => $upname,'goods_image_sort' => $pos + 1,'is_default' => $default));
+                            'store_id' => self::store_id,'goods_image' => $upname,'goods_image_sort' => $pos + 1,'is_default' => $default,'filesig' => $file_sig));
                     }
                 }
             }
@@ -501,26 +519,27 @@ class product_importer
                 }
             }
         }
-    }
-
-    public function proc()
-    {
-        $bodys = $this->tm_goods->field('num_iid,shop_id,body,description')->where(array('imported' => 0))->limit(false)->order('num_iid')->select();
 
-        $items = $this->get_tmgoods($bodys);
-        foreach($items as $item) {
-            print "handle num_iid=" . $item['num_iid'] . " start.\r\n";
-            $this->import_item($item['body'],$item['desc'],$item['num_iid'],$item['shop_id']);
-
-            $this->tm_goods->where(array('num_iid' => $item['num_iid']))->update(array('imported' => 1));
-            print "handle num_iid=" . $item['num_iid'] . " end. \r\n";
-        }
+        return $common_id;
     }
 
+//    public function proc()
+//    {
+//        $bodys = $this->tm_goods->field('num_iid,shop_id,body,description')->where(array('imported' => 0))->limit(false)->order('num_iid')->select();
+//
+//        $items = $this->get_tmgoods($bodys);
+//        foreach($items as $item) {
+//            print "handle num_iid=" . $item['num_iid'] . " start.\r\n";
+//            $this->import_item($item['body'],$item['desc'],$item['num_iid'],$item['shop_id']);
+//
+//            $this->tm_goods->where(array('num_iid' => $item['num_iid']))->update(array('imported' => 1));
+//            print "handle num_iid=" . $item['num_iid'] . " end. \r\n";
+//        }
+//    }
+
     public function proc_shop()
     {
-        //SELECT shop_id,count(*) FROM 33hao.lrlz_tm_goods group by shop_id;
-
+        ob_implicit_flush(1);
         $shopes = $this->tm_goods->field('shop_id,count(*) as n')->group('shop_id')->limit(false)->select();
 
         foreach($shopes as $shop)
@@ -528,42 +547,80 @@ class product_importer
             $shopid = $shop['shop_id'];
             $count = $shop['n'];
 
-            $bodys = $this->tm_goods->field('num_iid,cid,shop_id,body,description')->where(array('imported' => 0,'shop_id' => $shopid))->limit(false)->order('num_iid')->select();
+            Log::endl(Log::DEBUG);
+            Log::record("begin shop_id={$shopid} spu count= {$count}",Log::DEBUG);
+
+            $bodys = $this->tm_goods->field('num_iid,cid,shop_id,body,description,picnum')->where(array('imported' => 0,'shop_id' => $shopid))->limit(false)->order('num_iid')->select();
 
             $items = $this->get_tmgoods($bodys);
-            foreach($items as $item) {
-                print "handle num_iid=" . $item['num_iid'] . " start.\r\n";
+            foreach($items as $item)
+            {
+                $num_iid = $item['num_iid'];
+                $picnum = $item['picnum'];
 
-                $cid = $item['cid'];
-                if(is_excids($cid)) continue;
+                Log::endl(Log::DEBUG);
+                Log::record("handle num_iid={$num_iid} start.\r\n",Log::DEBUG);
 
-                $this->import_item($item['body'],$item['desc'],$item['num_iid'],$item['shop_id'],$nsku);
-                $this->tm_goods->where(array('num_iid' => $item['num_iid']))->update(array('imported' => 1));
-                print "handle num_iid=" . $item['num_iid'] . " end. \r\n";
+                echo "begin shop_id={$shopid} spu count= {$count} \r\n";
+                flush();
 
-                $num_iid = $item['num_iid'];
 
-                if($this->check_count($num_iid,$nsku) == false) {
-                    Log::record("shopid = {$shopid} and num_iid = {$num_iid} nsku = {$nsku} cannot be imported.");
+                $cid = $item['cid'];
+                if(is_excids($cid)) {
+                    --$count;
+                    continue;
+                }
+
+                $commonid =$this->import_item($item['body'],$item['desc'],$num_iid,$item['shop_id'],$nsku);
+                if($commonid)
+                {
+                    if($this->check($num_iid,$nsku,$picnum,$commonid) == false) {
+                        $this->mod_goods_attr_index->where(array('goods_commonid' => $commonid))->delete();
+                        $this->mod_goods_common->where(array('goods_commonid' => $commonid))->delete();
+                        $this->mod_goods_images->where(array('goods_commonid' => $commonid))->delete();
+                        $this->mod_goods->where(array('goods_commonid' => $commonid))->delete();
+                        Log::record("shopid = {$shopid} and num_iid = {$num_iid} nsku = {$nsku} cannot be imported.",Log::ERR);
+                    } else {
+                        $this->tm_goods->where(array('num_iid' => $item['num_iid']))->update(array('imported' => 1,'commonid' => $commonid));
+                        --$count;
+                    }
                 }
+                else
+                {
+                    Log::record("goods_commonid is error num_iid={$item['num_iid']}.",Log::DEBUG);
+                }
+
+                Log::record("handle num_iid={$item['num_iid']} end.",Log::DEBUG);
+
+                echo "handle num_iid={$item['num_iid']} end.";
+                flush();
             }
+
+            Log::record("end shop_id = {$shopid} Error= {$count}",Log::DEBUG);
         }
     }
 
-    private function check_count($num_iid,$nsku)
+    private function check($num_iid,$nsku,$picnum,$commonid)
     {
+        $ret = true;
         $ncommon = $this->mod_goods_common->where(array('num_iid' => $num_iid))->count();
         if($ncommon != 1) {
             Log::record("goods_common table's data is not equal 1 where num_iid = {$num_iid}",Log::ERR);
-            return false;
+            if($ret) $ret = false;
         }
 
         $ngoods = $this->mod_goods->where(array('num_iid' => $num_iid))->count();
 
         if($ngoods != $nsku) {
             Log::record("goods table's data is not equal {$nsku} where num_iid = {$num_iid}",Log::ERR);
-            return false;
+            if($ret) $ret = false;
+        }
+
+        $npics = $this->mod_goods_images->where(array('goods_commonid' => $commonid))->count();
+        if($npics < $picnum) {
+            Log::record("imageerror: goods image import error where goods_commonid = {$commonid} picnum={$picnum} npics={$npics}",Log::ERR);
+            if($ret) $ret = false;
         }
-        return true;
+        return $ret;
     }
 }

crontab/crawl/upload_file.php

@@ -140,8 +140,6 @@ class upload_file
      */
     public function upfile($field)
     {
-
-
         //上传文件
         $this->upload_file = $_FILES[$field];
 

run.php

@@ -4,7 +4,7 @@
 define('BASE_ROOT_PATH',str_replace('\\','/',dirname(__FILE__)));
 
 require_once (BASE_ROOT_PATH . '/fooder.php');
-$gfcgi_run = 1;
+$gfcgi_run = 2;
 
 if($gfcgi_run == 0) { //模拟请求
     $file = BASE_ROOT_PATH . '/debug_run.php';

shop/control/flea_goods.php

@@ -49,7 +49,9 @@ class flea_goodsControl extends BaseHomeControl {
 		$model_store_goods	= Model('flea');
 		$goods_array		= $model_store_goods->listGoods(array('goods_id'=>intval($_GET['goods_id']),'goods_show'=>'1'),'','flea.*');
 		
-		if(empty($goods_array))showMessage($lang['goods_index_no_goods'],'','html','error');//'商品不存在'
+		if(empty($goods_array)) {
+			showMessage($lang['goods_index_no_goods'],'','html','error');
+		}//'商品不存在'
 		/**
 		 * 图片路径
 		 */