xyzshop/crontab/crawl/product_importer.php

<?php
/**
 * Created by PhpStorm.
 * User: stanley-king
 * Date: 15/11/2
 * Time: 下午2:59
 */

require_once(BASE_DATA_PATH . '/model/goods.model.php');


class product_importer
{
    private $tm_goods;
    private $mod_spec;
    private $mod_spec_value;

    private $mod_type_spec;
    private $tm_spvid_tpvid; //spec value id => tmall props vid

    private $mod_attribute;
    private $tm_cpid_attrid;
    private $mod_attribute_value;
    private $tm_tvid_attrvid;

    private $mod_goods_common;
    private $mod_goods;
    private $mod_goods_images;

    private $mod_goods_attr_index;

    private $cls_tree;
    private $down_path;

    public function __construct()
    {
        $this->tm_goods = Model('tm_goods');

        $this->mod_spec = Model('spec');
        $this->mod_good_class = Model('goods_class');
        $this->mod_spec_value = Model('spec_value');

        $this->mod_type = Model('type');
        $this->mod_type_spec = Model('type_spec');
        $this->tm_spvid_tpvid = Model('tm_spvid_tpvid');

        $this->mod_attribute = Model('attribute');
        $this->tm_cpid_attrid = Model('tm_cpid_attrid');
        $this->mod_attribute_value = Model('attribute_value');
        $this->tm_tvid_attrvid = Model('tm_tvid_attrvid');
        $this->mod_goods_images = Model('goods_images');

        $this->mod_goods = new goodsModel();
        $this->mod_goods_common = Model('goods_common');
        $this->cls_tree = new class_tree();

        $this->mod_goods_attr_index = Model('goods_attr_index');

        $this->down_path = BASE_DATA_PATH . '/Download';
    }

    private function get_tmgoods($bodys)
    {
        $result = array();

        foreach($bodys as $body) {
            $item = json_decode($body['body'],true);
            $product = array();
            $product['body'] = $item['item_seller_get_response']['item'];
            $product['desc'] = $body['description'];
            $product['num_iid'] = $body['num_iid'];
            $product['shop_id'] = $body['shop_id'];
            $product['picnum'] = $body['picnum'];

            array_push($result,$product);
        }

        return $result;
    }



    private function fill_class($cid,&$general,&$name)
    {
        $this->cls_tree->get_info($cid,$cid_1,$cid_2,$cid_3,$name);
        $general['gc_id'] = $cid;
        $general['gc_id_1'] = $cid_1;
        $general['gc_id_2'] = $cid_2;
        $general['gc_id_3'] = $cid_3;
    }

    private function  get_sp_name($sp_id)
    {
        $items = $this->mod_spec->where(array('sp_id' => $sp_id))->limit(false)->select();
        foreach($items as $item) {
            $name = $item['sp_name'];
            return $name;
        }

        return '';
    }

    private function get_specar($typeid)
    {
        $items = $this->mod_type_spec->where(array("type_id" => $typeid))->limit(false)->select();

        $ret = array();
        foreach($items as $item)
        {
            $sp_id = $item['sp_id'];
            $sp_name = $this->get_sp_name($sp_id);
            $ret[$sp_id] = $sp_name;
        }

        return $ret;
    }

    private function get_spec_name($sp_id)
    {
        $items = $this->mod_spec->field('sp_name')->where(array('sp_id' => $sp_id))->limit(false)->select();
        if(empty($items)) {
            return '';
        } else {
            return $items[0]['sp_name'];
        }
    }

    /**
     * @param $spid
     * @param $spvid
     * @return mixed
     */
    private function get_specval_name($spid, $spvid)
    {
        $items = $this->mod_spec_value->field('sp_value_name')->where(array('sp_id' => $spid, 'sp_value_id' => $spvid))->limit(false)->select();
        if(empty($items)) {
            return '';
        } else {
            return $items[0]['sp_value_name'];
        }
    }

    private function get_spvid($cid,$pid,$vid)
    {
        $items = $this->tm_spvid_tpvid->field('spvid')->where(array('tpcid' => $cid,'tppid' => $pid,'tpvid' => $vid))->limit(false)->select();
        if(empty($items)) {
            return '';
        } else {
            return $items[0]['spvid'];
        }
    }

    private function get_attrid($cid,$pid)
    {
        $items = $this->tm_cpid_attrid->field('attr_id')->where(array('cid' => $cid,'pid' => $pid))->limit(false)->select();
        if(empty($items)) {
            return '';
        } else {
            return $items[0]['attr_id'];
        }
    }

    private function get_attrvid($cid,$pid,$vid)
    {
        $items = $this->tm_tvid_attrvid->field('attr_vid')->where(array('tcid' => $cid,'tpid' => $pid,'tvid' => $vid))->limit(false)->select();
        if(empty($items)) {
            return '';
        } else {
            return $items[0]['attr_vid'];
        }
    }

    private function parse_sku_properties($cid,$props,&$spec_name,&$spec_val)
    {
        $spec_vals = preg_split("/[;]+/", $props);

        foreach($spec_vals as $sv)
        {
            $data = preg_split("/[:]+/", $sv);
            if(!empty($data)) {
                $tmpid = $data[0];
                $tmpvid = $data[1];

                $spvid = $this->get_spvid($cid,$tmpid,$tmpvid);

                $spid = $tmpid;
                $sp_name = $this->get_spec_name($spid);
                $spval_name = $this->get_specval_name($spid, $spvid);

                $spec_val[$spid][$spvid] = $spval_name;
                $spec_name[$spid] = $sp_name;
            }
        }
    }

    private function get_attr_name($attrid)
    {
        $items = $this->mod_attribute->table('attribute')->field('attr_name')->where(array('attr_id' => $attrid))->limit(false)->select();
        if(empty($items)) {
            return NULL;
        } else {
            return $items[0]['attr_name'];
        }
    }

    private function get_attr_vname($typeid,$attrid,$attrvid)
    {
        $items = $this->mod_attribute_value->table('attribute_value')->field('attr_value_name')->where(array('attr_id' => $attrid,'type_id' => $typeid,'attr_value_id' => $attrvid))->limit(false)->select();
        if(empty($items)) {
            return NULL;
        } else {
            return $items[0]['attr_value_name'];
        }
    }

    private function get_common_attr($cid,$props)
    {
        $ar_attr= array();

        $spec_vals = preg_split("/[;]+/", $props);
        foreach($spec_vals as $sv)
        {
            $data = preg_split("/[:]+/", $sv);
            if(!empty($data))
            {
                $tmpid = $data[0];
                $tmvid = $data[1];

                $attr_id = $this->get_attrid($cid,$tmpid);
                if(!empty($attr_id)) {
                    $attr_name = $this->get_attr_name($attr_id);
                    $ar_attr[$attr_id]['name'] = $attr_name;
                    $attr_vid = $this->get_attrvid($cid,$tmpid,$tmvid);

                    $type_id = $cid;
                    $attr_vname = $this->get_attr_vname($type_id,$attr_id,$attr_vid);
                    $ar_attr[$attr_id][$attr_vid] = $attr_vname;
                }
            }
        }

        return $ar_attr;
    }

    private function get_common_spec_nameval($cid,$skus,&$spec_name,&$spec_val)
    {
        $ar_spec_name = array();
        $ar_spec_val = array();

        foreach($skus as $skuex)
        {
            foreach($skuex as $sku) {
                $properties = $sku['properties'];
                $this->parse_sku_properties($cid,$properties,$ar_spec_name,$ar_spec_val);
            }
        }

        $spec_name = serialize($ar_spec_name);
        $spec_val = serialize($ar_spec_val);
    }

    private function get_goods_spec($cid,$properties)
    {
        $ar_spec_name = array();
        $ar_spec_val = array();

        $this->parse_sku_properties($cid,$properties,$ar_spec_name,$ar_spec_val);

        $specs = array();
        foreach($ar_spec_val as $spec_val)
        {
            foreach($spec_val as $key => $val) {
                $specs[$key] = $val;
            }
        }

        return $specs;
    }

    private function fill_general($cid,$item, &$general,&$name,$num_iid,$shop_id)
    {
        $this->fill_class($cid, $general,$name);

        $general['num_iid'] = $num_iid;
        $general['tm_shop'] = $shop_id;
        $general['store_id'] =  fetch_config::store_id;
        $general['store_name'] = fetch_config::store_name;

        $general['goods_name'] = not_null($item['title']);
        $general['goods_mobile_name'] = not_null($item['title']);
        $general['goods_jingle'] = not_null($item['sell_point']);
        $general['goods_price'] = $item['price'];
        $general['goods_marketprice'] = $item['price'];
        $general['goods_image'] = $item['pic_url'];
        $general['goods_state'] = fetch_config::goods_state;
        $general['goods_verify'] = 1;
        $general['goods_commend'] = strtolower($item['has_showcase']) ? 1 : 0;
        $general['goods_vat'] = strtolower($item['has_invoice']) ? 1 : 0;
        $general['is_virtual'] = strtolower($item['is_virtual']) ? 1 : 0;
        $general['store_name'] = $item['nick'];
        $general['goods_serial_tm'] = not_null($item['outer_id']);
        $general['goods_addtime'] = strtotime($item['created']);
    }

    private function reset_imgs($imges)
    {
        if(empty($imges)) return '';

        $ret = array();

        foreach($imges as $img) {
            $ret[$img['position']]['url'] = $img['url'];
        }

        return $ret;
    }

    private function upload_imges(&$imges,$storeid)
    {
        foreach($imges as &$img)
        {
            static $pngext = array('png','jpg','jpeg');

            $url = $img['url'];
            $info = pathinfo($url);
            $src_name = $this->down_path . '/' . md5($url) . '.' . $info['extension'];

            if(empty($info['extension'])) continue;

            $err = true;
            if(file_exists($src_name))
            {
                $fh = fopen($src_name, "rb");
                $head = fread($fh, 8);
                fclose($fh);

                $err = false;
                if(empty($head)) {
                    unlink($src_name);
                    Log::record("{$src_name} file is empty.",Log::ERR);
                    $err = true;
                }

                if(in_array($info['extension'],$pngext))
                {
                    $arr = unpack("C4", $head);
                    if($arr[1] == 137 &&  $arr[2] == 80) {
                        Log::record("{$src_name} is a png file.",Log::ERR);
                        $err = true;
                    }
                }
            }
            if($err) {
                Log::record("{$src_name} file is error.",Log::ERR);
                continue;
            }

            $upctl = new upload_control();
            $upctl->set_store($storeid);
            $up_name = $upctl->upload_img($src_name);
            $img['upname'] = $up_name;
            $img['file'] = $src_name;
        }

        return true;
    }

    private function add_attr($gd_id,$gdcommon_id,$gc_id,$type_id,$goods_attr)
    {
        foreach($goods_attr as $attrid => $values)
        {
            foreach($values as $vid => $vname)
            {
                if($vid != 'name')
                {
                    $index = $this->mod_goods_attr_index->insert(array('goods_id' => $gd_id,'goods_commonid' => $gdcommon_id,
                        'gc_id' => $gc_id,'type_id' => $type_id, 'attr_id' => $attrid,'attr_value_id' => $vid));
                    if($index == false) {
                        Log::record('Cannot insert attr_index');
                    }
                }
            }
        }
    }

    private function color_spec()
    {
        return 1627207;
    }

    private function goods_colorid($cid,$properties)
    {
        $data = preg_split("/[:]+/", $properties);
        if(!empty($data))
        {
            $tm_pid = $data[0];
            $tm_pvid = $data[1];
            if($tm_pid != $this->color_spec()) {
                return 0;
            } else {
                return $this->get_spvid($cid,$tm_pid,$tm_pvid);
            }
        }

        return 0;
    }

    private function import_item($item,$desc,$num_iid,$shop_id,&$nsku)
    {
        $cid = $item['cid'];

        $general = array();
        $this->fill_general($cid,$item,$general,$name,$num_iid,$shop_id);

        //属于SKU 信息中
        $skus = $item['skus'];
        $typeid = $cid;

        $goods_attr = $this->get_common_attr($cid,$item['props']);
        $goods_common = $general;
        {
            $goods_common['type_id'] = $typeid;
            $goods_common['goods_body'] = $desc;
            $goods_common['mobile_body'] = $desc;
            $goods_common['gc_name'] = $name;
            $goods_common['goods_selltime'] = strtotime($item['list_time']);
            $goods_common['goods_costprice'] = $item['price'];
            $goods_common['goods_attr'] = serialize($goods_attr);

            if(empty($skus)) {
                $spec_name = 'N';
                $spec_val = 'N';
            } else {
                $this->get_common_spec_nameval($cid,$skus,$spec_name,$spec_val);
            }

            $goods_common['spec_name'] =  $spec_name;
            $goods_common['spec_value'] = $spec_val;

            $common_id = $this->mod_goods->addGoodsCommon($goods_common);

            $prop_imgs = $item['prop_imgs']['prop_img'];
            if(!empty($prop_imgs)) {
                $this->upload_imges($prop_imgs,fetch_config::store_id);
            }

            $imgs = $this->reset_imgs($item['item_imgs']['item_img']);
            $this->upload_imges($imgs,fetch_config::store_id);

            if(empty($prop_imgs))
            {
                foreach($imgs as $pos => $img)
                {
                    $upname = $img['upname'];
                    if(empty($upname)) continue;
                    if(empty($main_pic)) {
                        $main_pic = $upname;
                        $default = 1;
                    } else {
                        $default = 0;
                    }

                    $file_sig = '';
                    if(isset($img['file'])) {
                        $file_sig = md5_file($img['file']);
                    }

                    $this->mod_goods_images->insert(array('goods_commonid' => $common_id,
                        'store_id' => fetch_config::store_id,'goods_image' => $upname,'goods_image_sort' => $pos,'is_default' => $default,'filesig' => $file_sig));
                }
            }
            else
            {
                foreach($prop_imgs as $pimg)
                {
                    $default = 1;
                    $pos = 0;
                    $pupname = $pimg['upname'];
                    if(empty($pupname)) continue;

                    $file_sig = '';
                    if(isset($pimg['file'])) {
                        $file_sig = md5_file($pimg['file']);
                    }

                    $color_id = $this->goods_colorid($cid,$pimg['properties']);
                    $this->mod_goods_images->insert(array('goods_commonid' => $common_id,'color_id' => $color_id,
                        'store_id' => fetch_config::store_id,'goods_image' => $pupname,'goods_image_sort' => $pos,'is_default' => $default,'filesig' => $file_sig));
                    $main_pic = $pupname;

                    $default = 0;
                    foreach($imgs as $pos => $img)
                    {
                        $upname = $img['upname'];
                        if(empty($upname)) continue;

                        $file_sig = '';
                        if(isset($img['file'])) {
                            $file_sig = md5_file($img['file']);
                        }
                        $this->mod_goods_images->insert(array('goods_commonid' => $common_id,'color_id' => $color_id,
                            'store_id' => fetch_config::store_id,'goods_image' => $upname,'goods_image_sort' => $pos + 1,'is_default' => $default,'filesig' => $file_sig));
                    }
                }
            }

            $this->mod_goods_common->where(array('goods_commonid' => $common_id))->update(array('goods_image' => $main_pic));
        }

        if(empty($skus))
        {
            $nsku = 1;
            $goods = $general;

            $goods['goods_edittime'] = strtotime($item['modified']);
            $goods['goods_serial_tm'] = not_null($item['outer_id']);
            $goods['goods_price'] = $item['price'];
            $goods['goods_addtime'] = strtotime($item['created']);
            $goods['goods_storage'] = empty($item['quantity']) ? 0 : $item['quantity'];
            $goods['goods_commonid'] = $common_id;
            //$goods['color_id'] = $this->goods_colorid($cid,$item['properties']);
            $goods['goods_spec'] = 'N';
            $goods['goods_image'] = $main_pic;

            $gd_id = $this->mod_goods->addGoods($goods);
            $this->add_attr($gd_id,$common_id,$cid,$typeid,$goods_attr);
        }
        else
        {
            foreach($skus as $skuex)
            {
                $nsku = count($skuex);
                foreach($skuex as $sku)
                {
                    $goods = $general;

                    $goods['goods_edittime'] = strtotime($sku['modified']);
                    $goods['goods_price'] = $sku['price'];
                    $goods['goods_addtime'] = strtotime($sku['created']);
                    $goods['goods_storage'] = $sku['quantity'];
                    $goods['goods_commonid'] = $common_id;
                    $goods_spec = $this->get_goods_spec($cid,$sku['properties']);
                    $goods['goods_spec'] = serialize($goods_spec);
                    $goods['goods_image'] = $main_pic;
                    $goods['color_id'] = $this->goods_colorid($cid,$sku['properties']);
                    $goods['goods_serial_tm'] = not_null($sku['outer_id']);


                    $gd_id = $this->mod_goods->addGoods($goods);
                    $this->add_attr($gd_id,$common_id,$cid,$typeid,$goods_attr);
                }
            }
        }

        return $common_id;
    }

    public function proc()
    {
        $condition = array('num_iid' => 13898341378);
        //$condition = array('imported' => 0);
        $bodys = $this->tm_goods->field('num_iid,shop_id,body,description')->where($condition)->limit(false)->order('num_iid')->select();

        $items = $this->get_tmgoods($bodys);
        foreach($items as $item) {
            print "handle num_iid=" . $item['num_iid'] . " start.\r\n";
            $this->import_item($item['body'],$item['desc'],$item['num_iid'],$item['shop_id']);

            $this->tm_goods->where(array('num_iid' => $item['num_iid']))->update(array('imported' => 1));
            print "handle num_iid=" . $item['num_iid'] . " end. \r\n";
        }
    }

    private function goods_exist($num_iid)
    {
        $items = $this->mod_goods->where(array('num_iid' => $num_iid))->select();
        if(!empty($items) && count($items) > 0) {
            return true;
        } else {
            return false;
        }
    }

    public function proc_shop()
    {
        $shopes = $this->tm_goods->field('shop_id,count(*) as n')->group('shop_id')->limit(false)->select();

        foreach($shopes as $shop)
        {
            $shopid = $shop['shop_id'];
            $count = $shop['n'];

            Log::record("begin shop_id={$shopid} spu count= {$count}",Log::DEBUG);

            $con = array('imported' => 0,'shop_id' => $shopid);
            $bodys = $this->tm_goods->field('num_iid,cid,shop_id,body,description,picnum')->where($con)->limit(false)->order('num_iid')->select();

            $items = $this->get_tmgoods($bodys);
            foreach($items as $item)
            {
                $num_iid = $item['num_iid'];
                $picnum = $item['picnum'];

                Log::record("handle num_iid={$num_iid} start.\r\n",Log::DEBUG);

                if($this->goods_exist($num_iid)) {
                    Log::record("handle num_iid={$item['num_iid']} has existed.",Log::DEBUG);
                    continue;
                }

                $cid = $item['cid'];
                if(is_excids($cid)) {
                    --$count;
                    Log::record("handle num_iid={$item['num_iid']} cid is exclude.",Log::DEBUG);
                    continue;
                }

                $commonid =$this->import_item($item['body'],$item['desc'],$num_iid,$item['shop_id'],$nsku);
                if($commonid)
                {
                    if($this->check($num_iid,$nsku,$picnum,$commonid) == false) {
                        $this->mod_goods_attr_index->where(array('goods_commonid' => $commonid))->delete();
                        $this->mod_goods_common->where(array('goods_commonid' => $commonid))->delete();
                        $this->mod_goods_images->where(array('goods_commonid' => $commonid))->delete();
                        $this->mod_goods->where(array('goods_commonid' => $commonid))->delete();
                        $this->mod_goods->where(array('num_iid' => $num_iid))->delete();
                        $this->mod_goods_common->where(array('num_iid' => $num_iid))->delete();

                        Log::record("shopid = {$shopid} and num_iid = {$num_iid} nsku = {$nsku} cannot be imported.",Log::ERR);
                    } else {
                        $this->tm_goods->where(array('num_iid' => $item['num_iid']))->update(array('imported' => 1,'commonid' => $commonid));
                        --$count;
                    }
                }
                else
                {
                    Log::record("goods_commonid is error num_iid={$item['num_iid']}.",Log::DEBUG);
                }

                Log::record("handle num_iid={$item['num_iid']} end.",Log::DEBUG);
            }

            Log::record("end shop_id = {$shopid} Error= {$count}",Log::DEBUG);
        }
    }

    private function check($num_iid,$nsku,$picnum,$commonid)
    {
        $ret = true;
        $ncommon = $this->mod_goods_common->where(array('num_iid' => $num_iid))->count();
        if($ncommon != 1) {
            Log::record("goods_common table's data is not equal 1 where num_iid = {$num_iid}",Log::ERR);
            if($ret) $ret = false;
        }

        $ngoods = $this->mod_goods->where(array('num_iid' => $num_iid))->count();

        if($ngoods != $nsku) {
            Log::record("goods table's data is not equal {$nsku} where num_iid = {$num_iid}",Log::ERR);
            if($ret) $ret = false;
        }

        $npics = $this->mod_goods_images->where(array('goods_commonid' => $commonid))->count();
        if($npics < $picnum) {
            Log::record("imageerror: goods image import error where goods_commonid = {$commonid} picnum={$picnum} npics={$npics}",Log::ERR);
            if($ret) $ret = false;
        }
        return $ret;
    }
}