TF-IDF

#导入特征提取包，第一个用于计数，第二个用于计算词频

from sklearn.feature_extraction.text import CountVectorizer

from sklearn.feature_extraction.text import TfidfTransformer

x_train = ['TF-IDF 主要 思想 是',

           '算法 一个 重要 特点 可以 脱离 语料库 背景',

           '如果 一个 网页 被 很多 其他 网页 链接 说明 网页 重要']

x_test=['原始 文本 进行 标记',

        '主要 思想']

#该类会将文本中的词语转换为词频矩阵，矩阵元素 a [i][j] 表示 j 词在 i 类文本下的词频

vectorizer = CountVectorizer(max_features=10)

#该类会统计每个词语的 tf-idf 权值

tf_idf_transformer = TfidfTransformer()

#将文本转为词频矩阵并计算 tf-idf

tf_idf = tf_idf_transformer.fit_transform(vectorizer.fit_transform(x_train))

#将 tf-idf 矩阵抽取出来，元素 a [i][j] 表示 j 词在 i 类文本中的 tf-idf 权重

x_train_weight = tf_idf.toarray()

#对测试集进行 tf-idf 权重计算

tf_idf = tf_idf_transformer.transform(vectorizer.transform(x_test))

x_test_weight = tf_idf.toarray()  # 测试集 TF-IDF 权重矩阵

# 获取关键词集

def get_keywords(matrixArray, keywordsArray):

    return [keywordsArray[index] for index in range(len(matrixArray)) if matrixArray[index]>0] 

print('输出x_train文本向量：')

print(x_train_weight)

print('输出x_test文本向量：')

print(x_test_weight)

for i in range(len(x_test_weight)):

    print(f'测试集第{i}个文本的关键词是',get_keywords(x_test_weight[i], vectorizer.get_feature_names_out()))

# 详情页爬取

# requests 爬取网页

import requests

# 解析网页 

from lxml import etree

# 正则表达式

import re

# 将 dataframe (数据框) 导出成 excel

import pandas as pd

# 将代码执行过程以进度条展示

from tqdm import tqdm

tempdict = {

#     存储的标题

    'title': [],

#     没用

    'dtype': [],

#     目标网址链接

    'urlhref': []

}

headers = {

    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.102 Safari/537.36 Edg/98.0.1108.56'

}

#对网址发起请求

for i in tqdm(range(1,23)):

    baseurl = 'http://www.safehoo.com'

    url = baseurl + f'/Case/Case/Collapse/List_{i}.shtml'

    #网址先包含了对浏览器的检测，要加入 headers

    res = requests.get(url, headers=headers)

    #转化格式，防止乱码出现

    res.encoding = 'utf-8'

    html = etree.HTML(res.text)

    a = html.xpath('//div[@class="childclass_content"]/li')

    for i in a:

        # 标题，进行异常检测

        try:

            tempdict['title'].append(i.xpath('./a/text()')[0])

        except:

            tempdict['title'].append(i.xpath('./a/font/text()')[0])

        # 类型

        tempdict['dtype'].append(re.sub(' ','',re.sub('[A-Za-z0-9\!\%\[\]\,\。]','',i.xpath('./text()')[0])))

        # 地址

        tempdict['urlhref'].append(baseurl+i.xpath('./a/@href')[0])

#将数据转化为数据框格式

df = pd.DataFrame(tempdict)

#将数据保存到 excel 里面

df.to_excel('output.xlsx', index=False)

# 内容爬取

import re

import pandas as pd

import requests

from tqdm import tqdm

from lxml import etree

df = pd.read_excel('output.xlsx')

url = df['urlhref']

df['result'] = None

headers = {

    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.102 Safari/537.36 Edg/98.0.1108.56'

}

for i in tqdm(range(len(url))):

    res = requests.get(url[i], headers=headers)

    res.encoding = 'utf-8'

    html = etree.HTML(res.text)

    try:

        item = html.xpath('//*[@id="prt2"]/div')[0].xpath('string(.)')

        result = re.sub(' ', '', re.sub('\r\n ', '', item))

    except:

        result = None

    df['result'][i] = result

df.to_excel('result.xlsx', index=False)

def split_function(text):

    type_list = ["一", "二", "三", "四", "五", "六", "七", "八", "九", "十"]

    result = ""

    text = text.replace('\r','').replace('\n','').replace('\t','').replace('\u3000', '')

    for i in range(len(type_list)):

        try:

            temptext = re.findall(f"{type_list[i]}、(.*?){type_list[i+1]}、", text)[0]

            if "概况" in temptext or "结果" in temptext:

                result = result + "事故概况:" + temptext.split('。')[0]

            if "原因" in temptext:

                result = result + "事故原因" + temptext

        except:

            break

    if "概况" not in result:

        try:

            result = "事故概况：" + text.split("。")[0] + result

        except Exception as e:

            print(e)

    if "原因" not in result:

        try:

            result = result + "事故原因" + ''.join(text.split("。")[1:])

        except Exception as e:

            print(e)

    return result

def get_reason(text):

    try:

        if(text[1]==''):

            return text[2]

        else:

            return text[1]

    except:

        return None

# -*- coding: utf-8 -*-

import struct

import os

# 拼音表偏移

startPy = 0x1540;

# 汉语词组表偏移

startChinese = 0x2628;

# 全局拼音表

GPy_Table = {}

# 解析结果

# 元组 (词频，拼音，中文词组) 的列表

# 原始字节码转为字符串

def byte2str(data):

    pos = 0

    str = ''

    while pos < len(data):

        c = chr(struct.unpack('H', bytes([data[pos], data[pos + 1]]))[0])

        if c != chr(0):

            str += c

        pos += 2

    return str

# 获取拼音表

def getPyTable(data):

    data = data[4:]

    pos = 0

    while pos < len(data):

        index = struct.unpack('H', bytes([data[pos],data[pos + 1]]))[0]

        pos += 2

        lenPy = struct.unpack('H', bytes([data[pos], data[pos + 1]]))[0]

        pos += 2

        py = byte2str(data[pos:pos + lenPy]) 

        GPy_Table[index] = py

        pos += lenPy

# 获取一个词组的拼音

def getWordPy(data):

    pos = 0

    ret = ''

    while pos < len(data):

        index = struct.unpack('H', bytes([data[pos], data[pos + 1]]))[0]

        ret += GPy_Table[index]

        pos += 2

    return ret

# 读取中文表

def getChinese(data):

    GTable = []

    pos = 0

    while pos < len(data):

        # 同音词数量

        same = struct.unpack('H', bytes([data[pos], data[pos + 1]]))[0]

        # 拼音索引表长度

        pos += 2

        py_table_len = struct.unpack('H', bytes([data[pos], data[pos + 1]]))[0]

        # 拼音索引表

        pos += 2

        py = getWordPy(data[pos: pos + py_table_len])

        # 中文词组

        pos += py_table_len

        for i in range(same):

            # 中文词组长度

            c_len = struct.unpack('H', bytes([data[pos], data[pos + 1]]))[0]

            # 中文词组

            pos += 2

            word = byte2str(data[pos: pos + c_len])

            # 扩展数据长度

            pos += c_len

            ext_len = struct.unpack('H', bytes([data[pos], data[pos + 1]]))[0]

            # 词频

            pos += 2

            count = struct.unpack('H', bytes([data[pos], data[pos + 1]]))[0]

            # 保存

            GTable.append((count, py, word))

            # 到下个词的偏移位置

            pos += ext_len

    return GTable

def scel2txt(file_name):

    print('-' * 60)

    with open(file_name, 'rb') as f:

        data = f.read()

    print("词库名：", byte2str(data[0x130:0x338])) # .encode('GB18030')

    print("词库类型：", byte2str(data[0x338:0x540]))

    print("描述信息：", byte2str(data[0x540:0xd40]))

    print("词库示例：", byte2str(data[0xd40:startPy]))

    getPyTable(data[startPy:startChinese])

    getChinese(data[startChinese:])

    return getChinese(data[startChinese:])

# scel 所在文件夹路径

in_path = './专业词词库'

# 输出词典所在文件夹路径

out_path = './词库'

fin = [fname for fname in os.listdir(in_path) if fname[-5:] == ".scel"]

for f in fin:

    try:

        for word in scel2txt(os.path.join(in_path, f)):

            file_path=(os.path.join(out_path, str(f).split('.')[0] + '.txt'))

            # 保存结果

            with open(file_path,'a+',encoding='utf-8')as file:

                file.write(word[2] + '\n')

        os.remove(os.path.join(in_path, f))

    except Exception as e:

        print(e)

        pass

#TfidfVectorizer 为上述 CountVectorizer,TfidfTransformer 的集合

vector = TfidfVectorizer(stop_words=stoplist, vocabulary = vocab)

tf_idf = vector.fit_transform(corpus)