Python机器学习NLP自然语言处理基本操作词袋模型

import jieba
from gensim import corpora
# 定义标点符号
punctuation = ["，", "。", "：", "；", "?", "!"]
# 定义语料
content = [
    "今天天气真不错!",
    "明天要下雨?",
    "后天要打雷。"
]
# 分词
seg = [jieba.lcut(con) for con in content]
print("语料:", seg)

# 去除标点符号
tokenized = seg.copy()
for s in tokenized:
    for p in punctuation:
        if p in s:
            s.remove(p)
print("去除标点:", tokenized)
# tokenized是去标点之后的
dictionary = corpora.Dictionary(seg)
print("词袋模型:", dictionary)

# 保存词典
dictionary.save("deerwester.dict")
# 查看字典和下标id的映射
print("编号:", dictionary.token2id)

Building prefix dict from the default dictionary ...
Loading model from cache C:UsersWindowsAppDataLocalTempjieba.cache
Loading model cost 1.140 seconds.
Prefix dict has been built successfully.
语料: [["今天天气", "真不错", "!"], ["明天", "要", "下雨", "?"], ["后天", "要", "打雷", "。"]]
去除标点: [["今天天气", "真不错"], ["明天", "要", "下雨"], ["后天", "要", "打雷"]]
词袋模型: Dictionary(7 unique tokens: ["今天天气", "真不错", "下雨", "明天", "要"]...)
编号: {"今天天气": 0, "真不错": 1, "下雨": 2, "明天": 3, "要": 4, "后天": 5, "打雷": 6}