对TXT文档中文分词积累分词库

#! usr/bin/env python
#coding=utf-8

import jieba

class WordCut:
    def __init__(self,sentence):
        # 初始化
        sentence = []
        
    def ConcatSentences(self,sentence):
        return','.join(sentence.values)
    
    #创建停用词表
    def stopwordslist(self,):
        stopwords = [line.strip() for line in open('D:/jupyter/dic_stop.txt',encoding='UTF-8').readlines()]
        return stopwords
    
    # 对句子进行中文分词
    def seg_depart(self,sentence):
        # 对文档中的每一行进行中文分词
        print("(%s):正在分词" % "**")
        sentence_depart = jieba.cut(sentence.strip())
        # 创建一个停用词列表
        stopwords = stopwordslist()
        # 输出结果为outstr
        outstr = ''
        # 去停用词
        for word in sentence_depart:
            if word not in stopwords:
                if word != '\t':
                    outstr += word
                    outstr += " "
        return outstr
    
    def wordtank(self, sentence):
        #将word_list列以‘ ’分开，分成多列
        word_list_df = word_list.str.split(' ',expand = True)

#将列转换为行
word_list_df_0 = word_list_df.stack()

# 将所有的分词合并
words = []
for content in words:
    words.extend(content)
    
# 创建分词数据框
import pandas as pd
corpus = pd.DataFrame(word_list_df_0, columns=['word'])
corpus['cnt'] = 1

# 分组统计
g = corpus.groupby(['word']).agg({'cnt': 'count'}).sort_values('cnt', ascending=False)
g = g.reset_index()

本文来自互联网用户投稿，文章观点仅代表作者本人，不代表本站立场，不承担相关法律责任。如若转载，请注明出处。 如若内容造成侵权/违法违规/事实不符，请点击【内容举报】进行投诉反馈！