package com.softwarebr.nunu.service;

import com.aliyun.oss.model.PutObjectResult;
import com.brframework.common.utils.DateTimeUtils;
import com.brframework.commonoss.core.AliYunOSSClient;
import com.brframework.commonweb.exception.HandleException;
import com.google.common.collect.Lists;
import com.softwarebr.nunu.service.go.PolicyService;
import lombok.extern.slf4j.Slf4j;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.nodes.Node;
import org.jsoup.select.Elements;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;

import java.io.DataInputStream;
import java.io.IOException;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.List;
import java.util.UUID;
import java.util.stream.Collectors;

import static com.brframework.common.utils.DateTimeUtils.FORMATTER_DEFAULT_YMDHMS;

/**
 * @Author: ljr
 * @Date: 2019/11/13 10:35
 * @Description:
 */
@Service
@Slf4j
public class SpiderService {


    /**
     * 佛山南海人民政府网
     */
    public static final String SPIDER_URL_BASEURL_TYPE_ONE = "http://www.nanhai.gov.cn";
    /**
     * 佛山南海人民政府网 区发展和改革局
     */
    public static final String SPIDER_URL_TYPE_ONE_OF_ONE = "http://www.nanhai.gov.cn/fsnh/wzjyh/fgj/zcwj/";
    /**
     * 佛山南海人民政府网 人力资源和社会保障局（含区社会保险基金管理局）
     */
    public static final String SPIDER_URL_TYPE_ONE_OF_TWO = "http://www.nanhai.gov.cn/fsnh/wzjyh/rsj/tzgg/";
    /**
     * 佛山南海人民政府网 区科学技术局
     */
    //public static final String SPIDER_URL_TYPE_ONE_OF_THREE = "http://www.nanhai.gov.cn/cms/html/71044/column_71044_1.html";
    public static final String SPIDER_URL_TYPE_ONE_OF_THREE = "http://www.nanhai.gov.cn/fsnh/wzjyh/kjj/zcfg/";
    /**
     * 佛山南海人民政府网 区经济促进局
     */
    public static final String SPIDER_URL_TYPE_ONE_OF_FOUR = "http://www.nanhai.gov.cn/fsnh/wzjyh/jmj/zcfg/";
    /**
     * 广东省工业和信息化厅
     */
    public static final String SPIDER_URL_TYPE_TWO_OF_ONE = "http://gdii.gd.gov.cn/zwgk/tzgg1011/index.html";


    @Autowired
    PolicyService policyService;

    @Autowired
    AliYunOSSClient ossClient;

    /**
     * 爬取佛山南海人民政府网的数据,type1的那些
     *
     * @param url
     * @param column 是否为分栏
     * @return
     */
    public Document getTopHtmlContentByUrlForUrlOne(String url, boolean column) {
        try {
            Document document = Jsoup.connect(url).userAgent("Mozilla").timeout(30000).get();
            System.out.println(document);
            List<Elements> allElements = Lists.newArrayList();
            if (column) {
                Elements columnElements = document.getElementsByClass("erji_lanmu");
                for (Element element : columnElements) {
                    List<String> imgPath = element.getElementsByTag("a").eachAttr("href");
                    System.out.println(element);
                    for (String singleColumn : imgPath) {
                        //每个栏目处理失败不应该影响其他栏目
                        try {
                            Document childDocument = Jsoup.connect(singleColumn).userAgent("Mozilla").timeout(30000).get();
                            allElements.add(childDocument.getElementsByClass("list_li"));
                        } catch (IOException e) {
                            log.error("获取分栏失败:", e);
                            log.error("获取分栏失败,url:" + url + ",栏目:" +  singleColumn);
                            continue;
                        }
                    }
                }
            }else {
                allElements.add(document.getElementsByClass("list_li"));
            }
            for (Elements elements : allElements) {
                for (Element element : elements) {
                    for (Node node : element.childNodes()) {
                        if (node.childNodeSize() > 0) {
                            String relativeUrl = node.childNode(0).attr("href");
                            try {
                                getDetailHtmlContentByUrlForUrlOne(relativeUrl);

                            } catch (Exception e) {
                                log.error("爬取url出错:", e);
                            }
                        }
                    }
                }
            }
            return document;
        } catch (IOException e) {
            log.error("爬取url出错:", e);
            throw new RuntimeException(e);
        }
    }

    /**
     * 每一个子链接的内容
     *
     * @param url
     * @return
     */
    public void getDetailHtmlContentByUrlForUrlOne(String url) {
        try {
            Document document = Jsoup.connect(url).userAgent("Mozilla").timeout(30000).get();
            String title = getTitleForUrlOne(document);
            String time = getTimeForUrlOne(document);
            //数据重复爬取检查放外面可以减少文件图片内存资源占用
            if (policyService.checkPolicy(url, title)) {
                log.info("数据重复爬取");
                return;
            }
            String content = getContentForUrlOne(document);
            policyService.saveSpiderPolicy(url, title, content, DateTimeUtils.stringToDateTime(time));
        } catch (IOException e) {
            log.error("爬取url出错:", e);
            throw new HandleException("爬取url出错");
        }
    }

    /**
     * 获取内容
     *
     * @param document
     * @return
     */
    public String getContentForUrlOne(Document document) {
        Elements text_gy_wenz = document.getElementsByClass("text_gy_wenz");
        Element element = text_gy_wenz.get(0);
        element.childNode(1).remove();
        getElementImage(element, SPIDER_URL_BASEURL_TYPE_ONE);
        getElementFileHref(element, SPIDER_URL_BASEURL_TYPE_ONE);
        StringBuilder builder = new StringBuilder();
        element.childNodes().forEach(pLabel -> builder.append(pLabel));
        String content = builder.toString();
        return content;
    }

    /**
     * 获取发布时间
     *
     * @param document
     * @return
     */
    public String getTimeForUrlOne(Document document) {
        Elements time_gy_wenz = document.getElementsByClass("time_gy_wenz");
        String[] timeElement = time_gy_wenz.html().split(" ");
        String time = timeElement[1] + " " + timeElement[2];
        return time;
    }

    /**
     * 获取标题
     *
     * @param document
     * @return
     */
    public String getTitleForUrlOne(Document document) {
        Elements title_gy_wenz = document.getElementsByClass("title_gy_wenz");
        Element titleElement = title_gy_wenz.get(0);
        String title = titleElement.html();
        return title;
    }

    /**
     * 爬取广东省工业和信息化厅的数据 http://gdii.gd.gov.cn/zwgk/tzgg1011/index.html
     * 标题和时间在首页爬取,内容及发布人在详情里面爬取
     */
    public void savePolicyForUrlTwo() {
        try {
            Document document = Jsoup.connect(SPIDER_URL_TYPE_TWO_OF_ONE).userAgent("Mozilla").timeout(30000).get();
            Elements elements = document.getElementsByClass("NewsList removebr");
            for (Node node : elements.get(0).childNodes()) {
                if (node.childNodeSize() > 0) {
                    try {
                        String time = node.childNode(0).childNode(0).toString();
                        String title = node.childNode(1).attr("title");
                        String href = node.childNode(1).attr("href");
                        if (title.contains("<br>")) {
                            title = title.substring(0, title.indexOf("<br>"));
                        }
                        getHtmlByUrlForUrlTwo(href, time, title);
                    } catch (Exception e) {
                        log.error("爬取广东省工业和信息化厅的数据出错:", e);
                        continue;
                    }

                }

            }
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    /**
     * 获取子级链接的内容
     *
     * @param url
     * @param time
     * @param title
     */
    public void getHtmlByUrlForUrlTwo(String url, String time, String title) {
        try {
            //数据重复爬取检查放外面可以减少文件图片内存资源占用
            if (policyService.checkPolicy(url, title)) {
                log.info("数据重复爬取");
                return;
            }
            Document document = Jsoup.connect(url).userAgent("Mozilla").timeout(30000).get();
            //标题和时间在上一级拿
            Element zoom = document.getElementById("zoom");
            getElementImage(zoom, "");
            getElementFileHref(zoom, "");
            String content = elementToContent(zoom);
            policyService.saveSpiderPolicy(url, title, content, DateTimeUtils.stringToDateTime(time.trim() + " 00:00:00", FORMATTER_DEFAULT_YMDHMS));
        } catch (IOException e) {
            log.error("爬取广东省工业和信息化厅政策失败,地址:" + url + ",错误原因:", e);
        }
    }

    private String elementToContent(Element element) {
        StringBuilder builder = new StringBuilder();
        element.childNodes().forEach(pLabel -> builder.append(pLabel));
        return builder.toString();
    }

    /**
     * 下载爬虫img标签图片文件并替换
     *
     * @param element
     * @throws IOException
     */
    private void getElementImage(Element element, String prefix) {
        try {
            List<String> imgPath = element.getElementsByTag("img").eachAttr("src");
            imgPath = imgPath.stream().map( path ->  path.indexOf("http") != -1 ? path : prefix + path).collect(Collectors.toList());
            imgPath = downFileUpload(imgPath);
            for (int i = 0; i < imgPath.size(); i++) {
                element.getElementsByTag("img").eq(i).attr("src", imgPath.get(i));
            }
        } catch (IOException e) {
            log.error("爬虫处理图片出现错误:", e);
        }
    }

    /**
     * 下载爬虫a标签文件并替换
     *
     * @param element 正文子节点
     * @param prefix 前缀,南海政府的是相对路径,要补成绝对路径
     */
    private void getElementFileHref(Element element, String prefix) {
        try {
            List<String> filePath = element.getElementsByAttributeValue("target", "_blank").eachAttr("href");
            filePath = filePath.stream().map(path -> path.indexOf("http") != -1 ? path : prefix + path).collect(Collectors.toList());
            filePath = downFileUpload(filePath);
            for (int i = 0; i < filePath.size(); i++) {
                element.getElementsByAttributeValue("target", "_blank").eq(i).attr("href", filePath.get(i));
            }
        } catch (IOException e) {
            log.error("爬虫处理文件出现错误:", e);
        }
    }

    /**
     * 下载文件上传
     *
     * @param filePath 文件地址
     * @return
     * @throws IOException
     */
    public List<String> downFileUpload(List<String> filePath) throws IOException {
        String bucket = ossClient.getDefaultBucketName();
        List<String> newFilePath = Lists.newArrayList();
        if (filePath.size() > 0) {
            for (String path : filePath) {
                URL url = new URL(path);
                HttpURLConnection urlCon = (HttpURLConnection) url.openConnection();
                urlCon.setConnectTimeout(6000);
                urlCon.setReadTimeout(6000);
                int code = urlCon.getResponseCode();
                if (code != HttpURLConnection.HTTP_OK) {
                    log.info("{0}文件读取失败", path);
                    return Lists.newArrayList();
                }
                String fileName = UUID.randomUUID().toString();
                String fileType = path.substring(path.lastIndexOf("."));
                // 读文件流
                DataInputStream in = new DataInputStream(urlCon.getInputStream());
                PutObjectResult result = ossClient.putObject(bucket, "nunu/" + fileName + fileType, in);
                newFilePath.add(ossClient.genUrl("nunu/" + fileName + fileType));
            }
        }
        return newFilePath;
    }


}
