Sfoglia il codice sorgente

modify down tmdata

stanley-king 9 anni fa
parent
commit
58430cab3f

+ 14 - 6
crontab/crawl/fetch_goods.php

@@ -17,6 +17,7 @@ require_once (BASE_CRONTAB_PATH . '/crawl/page_request.php');
 require_once (BASE_CRONTAB_PATH . '/crawl/class_tree.php');
 require_once (BASE_CRONTAB_PATH . '/crawl/pretreat_tmdata.php');
 require_once (BASE_CRONTAB_PATH . '/crawl/spec_attr.php');
+require_once (BASE_CRONTAB_PATH . '/crawl/spec_attr_parser.php');
 require_once (BASE_CRONTAB_PATH . '/crawl/upload_file.php');
 require_once (BASE_CRONTAB_PATH . '/crawl/upload_control.php');
 require_once (BASE_CRONTAB_PATH . '/crawl/tm_pagereq.php');
@@ -24,6 +25,15 @@ require_once (BASE_CORE_PATH . '/framework/libraries/uploadfile.php');
 
 date_default_timezone_set('Asia/Shanghai');
 
+function is_excids($cid)
+{
+    //50023728 赠品
+    //50023725 邮费
+    //50025832 搭配商品
+    static $cids = array(50023725,50023728,50025832);
+    return in_array($cid,$cids);
+}
+
 $tmreq = new tmrequest();
 $tmreq->proc();
 
@@ -32,11 +42,9 @@ $cat->proc();
 
 $treater = new pretreat_tmdata();
 $treater->proc();
-//
-$spec = new spec_attr();
-$spec->proc();
-
-//$importer = new product_importer();
-//$importer->proc();
 
+$spec = new spec_attr_parser();
+$spec->proc();
 
+$importer = new product_importer();
+$importer->proc();

+ 3 - 1
crontab/crawl/product_importer.php

@@ -63,7 +63,7 @@ class product_importer
 
     private function get_tmgoods()
     {
-        $bodys = $this->tm_goods->field('num_iid,body,description')->where(array('imported' => 0,'num_iid' => 39955135116))->limit(false)->order('num_iid')->select();
+        $bodys = $this->tm_goods->field('num_iid,body,description')->where(array('imported' => 0))->limit(false)->order('num_iid')->select();
         $result = array();
 
         foreach($bodys as $body) {
@@ -501,6 +501,8 @@ class product_importer
                     $goods['goods_spec'] = serialize($goods_spec);
                     $goods['goods_image'] = $main_pic;
                     $goods['color_id'] = $this->goods_colorid($cid,$sku['properties']);
+                    $goods['goods_serial'] = $sku['outer_id'];
+
 
                     $gd_id = $this->mod_goods->addGoods($goods);
                     $this->add_attr($gd_id,$common_id,$cid,$typeid,$goods_attr);

+ 351 - 0
crontab/crawl/spec_attr_parser.php

@@ -0,0 +1,351 @@
+<?php
+
+/**
+ * Created by PhpStorm.
+ * User: stanley-king
+ * Date: 15/12/5
+ * Time: 下午4:21
+ */
+
+class spec_attr_parser
+{
+    private $tm_goods;
+    private $tm_spec;
+    private $tm_props;
+    private $tm_prop_values;
+
+    private $mod_spec;
+    private $mod_spec_value;
+    private $mod_good_class;
+
+    private $mod_type;
+    private $mod_type_spec;
+    private $tm_spvid_tpvid; //spec value id => tmall props vid
+
+    private $mod_attribute;
+    private $tm_cpid_attrid;
+    private $mod_attribute_value;
+    private $tm_tvid_attrvid;
+
+    const store_name = "美宝莲";
+    const store_id = 1;
+
+    public function __construct()
+    {
+        $this->tm_spec = Model('tm_spec');
+        $this->tm_props = Model('tm_props');
+        $this->tm_prop_values = Model('tm_prop_values');
+        $this->tm_goods = Model('tm_goods');
+
+        $this->mod_good_class = Model('goods_class');
+        $this->mod_spec = Model('spec');
+        $this->mod_type = Model('type');
+        $this->mod_type_spec = Model('type_spec');
+        $this->mod_spec_value = Model('spec_value');
+        $this->tm_spvid_tpvid = Model('tm_spvid_tpvid');
+
+        $this->mod_attribute = Model('attribute');
+        $this->tm_cpid_attrid = Model('tm_cpid_attrid');
+        $this->mod_attribute_value = Model('attribute_value');
+        $this->tm_tvid_attrvid = Model('tm_tvid_attrvid');
+    }
+
+
+    //根据天猫的商品规格,对应生成规格,并做好分类,类型,规格的映射关系
+    public function proc()
+    {
+        $this->add_spec(); //建立规格
+        $this->add_type(); //建立类型,以及 规格与类型 的关系
+        $this->add_type_spec();
+
+        $this->add_spec_val(); //添加规格值,由于规格值是按照分类走的
+        $this->add_attribute();
+        $this->update_attrval();
+
+    }
+
+    private function add_spec()
+    {
+        $datas = $this->tm_spec->group('props')->order('props')->limit(false)->select(); //获取所有独立的规格
+        foreach ($datas as $item) {
+            $cid = $item['cid'];
+            $pid = $item['props'];
+            $spec_name = $this->get_tm_props_name($cid, $pid);
+            $sp_id = $this->mod_spec->insert(array('sp_id' => $pid, 'sp_name' => $spec_name));
+            if($sp_id != $pid) {
+                Log::record(__FUNCTION__ . ':' . __LINE__ . "  cannt find prop values from $cid $pid",Log::ERR);
+            }
+        }
+    }
+
+    private function add_type()
+    {
+        //通过分类直接建立类型
+        $items = $this->tm_spec->field('cid')->group('cid')->limit(false)->select();
+        foreach($items as $item) {
+            $cid = $item['cid']; // $cid => $typeid 以分类建立类型
+            $cnmae = $this->get_class_name($cid);
+            $this->mod_type->insert(array("type_id" => $cid, "type_name" => $cnmae));
+            $this->mod_good_class->where(array("gc_id" => $cid))->update(array("type_id" => $cid, "type_name" => $cnmae));
+        }
+    }
+
+    private function type_spec_exist($type_id,$sp_id)
+    {
+        $items = $this->mod_type_spec->where(array('type_id' => $type_id,'sp_id' => $sp_id))->limit(false)->select();
+        return (!empty($items));
+    }
+
+    private function add_type_spec()
+    {
+        //按照分类建立类型
+        $items = $this->tm_spec->field('cid,props')->group('cid,props')->limit(false)->select();
+        foreach($items as $item)
+        {
+            $cid = $item['cid']; // $cid => $typeid 以分类建立类型
+            $pid = $item['props'];
+
+            if($this->type_spec_exist($cid,$pid)) continue;
+
+            $this->mod_type_spec->insert(array("type_id" => $cid,"sp_id" => $pid));
+        }
+    }
+
+    private function get_tmgoods()
+    {
+        $bodys = $this->tm_goods->field('num_iid,body')->where(array('imported' => 0))->limit(false)->order('num_iid')->select();
+        $result = array();
+
+        foreach($bodys as $body) {
+            $item = json_decode($body['body'],true);
+            if(empty($item)) {
+                Log::record('Cannot decode num_iid='.$body['num_iid'].' goods.');
+                continue;
+            }
+            $product = array();
+            $product['body'] = $item['item_seller_get_response']['item'];
+            $product['num_iid'] = $body['num_iid'];
+
+            $cid = $product['body']['cid'];
+
+            if(is_excids($cid)) continue;
+
+            array_push($result,$product);
+        }
+
+        return $result;
+    }
+
+    private function parse_properties_name($props)
+    {
+        $spec_vals = preg_split("/[;]+/", $props);
+
+        $ret = array();
+        foreach($spec_vals as $sv)
+        {
+            $data = preg_split("/[:]+/", $sv);
+            if(!empty($data)) {
+                $item = array();
+                $item['pid'] = $data[0];
+                $item['vid'] = $data[1];
+                $item['pname'] = $data[2];
+                $item['vname'] = $data[3];
+
+                if(empty($item['pid']) || empty($item['vid']) || empty($item['pname']) || empty($item['vname'])) {
+                    continue;
+                } else {
+                    array_push($ret,$item);
+                }
+            }
+        }
+        return $ret;
+    }
+
+    private function specval_exist($props,$cid)
+    {
+        $pid = $props['pid'];
+        $vid = $props['vid'];
+
+        $results = $this->tm_spvid_tpvid->where(array('tppid' => $pid,'tpvid' => $vid, 'tpcid' => $cid))->select();
+        return (!empty($results));
+    }
+
+    private function add_spval($props,$cid)
+    {
+        foreach($props as $spvals)
+        {
+            if($this->specval_exist($spvals,$cid)) continue;
+
+            $spec_val = array();
+            $spec_val['sp_id'] = $spvals['pid'];
+            $spec_val['sp_value_name'] = $spvals['vname'];
+            $spec_val['store_id'] = 1;
+            $spec_val['sp_value_color'] = '#ddd9c3';
+            $spec_val['gc_id'] = $cid;
+            $spec_val['sp_value_sort'] = 0;
+
+            $tm_vid = $spvals['vid'];
+            $specval_id = $this->mod_spec_value->insert($spec_val);
+
+            if($specval_id)
+            {
+                $ret = $this->tm_spvid_tpvid->insert(array('spvid' => $specval_id,'tpcid' => $cid, 'tppid' => $spvals['pid'],'tpvid' => $tm_vid));
+                if(empty($ret)) {
+                    Log::record(__FUNCTION__ . ':' . __LINE__,Log::ERR);
+                }
+            }
+            else
+            {
+                Log::record(__FUNCTION__ . ':' . __LINE__,Log::ERR);
+            }
+        }
+    }
+
+    private function add_spec_val()
+    {
+        $tmgoods = $this->get_tmgoods();
+        foreach ($tmgoods as $goods)
+        {
+            $cid = $goods['body']['cid'];
+            $skus = $goods['body']['skus'];
+            if(empty($skus)) continue;
+            $skus = $skus['sku'];
+            foreach($skus as $sku) {
+                $prpos_name = $this->parse_properties_name($sku['properties_name']);
+                $this->add_spval($prpos_name,$cid);
+            }
+        }
+    }
+
+    private function get_class_name($cid)
+    {
+        $item = $this->mod_good_class->where(array('gc_id' => $cid))->limit(false)->select();
+        if(empty($item)) {
+            return NULL;
+        } else {
+            return $item[0]['gc_name'];
+        }
+    }
+
+    //根据cid 和 pid 取到规格名称。
+    private function get_tm_props_name($cid,$pid)
+    {
+        $item = $this->tm_props->where(array('cid' => $cid,'pid' => $pid))->limit(false)->select();
+        if(empty($item)) {
+            return NULL;
+        } else {
+            return $item[0]['name'];
+        }
+    }
+
+    private function attrval_exist($props,$cid)
+    {
+        $pid = $props['pid'];
+        $vid = $props['vid'];
+
+        $results = $this->tm_tvid_attrvid->field('attr_vid')->where(array('tpid' => $pid,'tvid' => $vid, 'tcid' => $cid))->select();
+
+        if(empty($results)) {
+            return false;
+        } else {
+            return $results[0]['attr_vid'];
+        }
+    }
+
+    private function attr_exist($props,$cid)
+    {
+        $pid = $props['pid'];
+
+        $results = $this->tm_cpid_attrid->field('attr_id')->where(array('pid' => $pid, 'cid' => $cid))->select();
+        if(empty($results)) {
+            return false;
+        } else {
+            return $results[0]['attr_id'];
+        }
+    }
+
+    private function is_private_attr($pid,$cid)
+    {
+        if($pid == 20000) return false;
+        $items = $this->tm_props->field('id')->where(array('pid' => $pid,'cid' => $cid))->select();
+
+        return empty($items);
+    }
+
+
+    //需要从天猫的数据表里面排除掉 规格 =》 从而得到属性
+    public function add_attribute()
+    {
+        $tmgoods = $this->get_tmgoods();
+        foreach ($tmgoods as $goods)
+        {
+            $cid = $goods['body']['cid'];
+            $props = $this->parse_properties_name($goods['body']['props_name']);
+            $expids = $this->get_specids($cid);
+
+            foreach($props as $attr)
+            {
+                $pid = $attr['pid'];
+                if(in_array($pid,$expids)) continue;
+                if($this->is_private_attr($pid,$cid)) continue;
+
+                $attrid = $this->attr_exist($attr,$cid);
+                if($attrid == false) {
+                    $row = array("attr_name" => $attr['pname'],"type_id" => $cid,"attr_show" => 1);
+                    $attrid = $this->mod_attribute->table('attribute')->insert($row);
+                    $this->tm_cpid_attrid->insert(array("attr_id" => $attrid, "cid" => $cid, "pid" => $pid));
+                }
+
+                if($this->attrval_exist($attr,$cid)) continue;
+
+                $attr_vid = $this->mod_attribute_value->insert(array("attr_id" => $attrid,"type_id" => $cid, "attr_value_name" => $attr['vname']));
+                if($attr_vid)
+                {
+                    $rid = $this->tm_tvid_attrvid->insert(array('attr_vid' =>$attr_vid,'tcid' => $cid,'tpid' => $pid,'tvid' => $attr['vid']));
+                    if(empty($rid)) {
+                        throw new Exception('insert tm_tvid_attrvid error');
+                    }
+                }
+                else
+                {
+                    throw new Exception('insert attribute_value error');
+                }
+            }
+        }
+    }
+
+
+
+    private function get_specids($typeid)
+    {
+        $ret = array();
+        $items = $this->mod_type_spec->field('type_id,sp_id')->where(array('type_id' => $typeid))->limit(false)->select();
+        foreach($items as $item) {
+            array_push($ret,(int)$item['sp_id']);
+        }
+
+        return $ret;
+    }
+
+    private function attr_value($type_id,$attr_id)
+    {
+        $items = $this->mod_attribute_value->field('attr_value_id,attr_value_name')->where(array('attr_id' => $attr_id,'type_id' => $type_id))->order('attr_value_sort')->limit(false)->select();
+
+        $ret = array();
+        foreach($items as $item) {
+            array_push($ret,$item['attr_value_name']);
+        }
+        return implode(',',$ret);
+    }
+
+    private function update_attrval()
+    {
+        $items = $this->mod_attribute->table('attribute')->field('attr_id,type_id')->limit(false)->select();
+        foreach($items as $item) {
+            $attr_id = $item['attr_id'];
+            $type_id = $item['type_id'];
+            $attr_value = $this->attr_value($type_id,$attr_id);
+            $this->mod_attribute->table('attribute')->where(array('attr_id' => $attr_id))->update(array('attr_value' => $attr_value));
+        }
+    }
+}