批量将带中文的csv文件编码改成utf-8的方法

import os
import csv

# 指定目录
dir_path = "/Users/shibo/20230413/备份20230413/mooc/分类算法/5.XGBoost 分类算法电商购买意愿"

# 文件名列表
oldfile_names=[]
newfile_names=[]
file_Names = ['data_用户表','data_2月用户行为数据','data_3月用户行为数据','data_4月用户行为数据']
for i in file_Names:
    oldfile_name = i + ".csv"
    oldfile_names.append(oldfile_name)
    newfile_name = i + "_new.csv"
    newfile_names.append(newfile_name)

# 遍历文件名列表，创建空的CSV文件
for file_name in newfile_names:
    # 拼接出文件的完整路径
    file_path = os.path.join(dir_path, file_name)

    # 创建并打开CSV文件
    with open(file_path, mode='w', newline='') as csv_file:
        # 创建CSV写入器
        writer = csv.writer(csv_file)

        # 写入表头
        writer.writerow(["Header1", "Header2", "Header3"])

生成一系列空 Excel 备用。

import codecs
def handleEncoding(original_file,newfile):
    #newfile=original_file[0:original_file.rfind(.)]+'_copy.csv'
    f=open(original_file,'rb+')
    content=f.read()#读取文件内容，content为bytes类型，而非string类型
    source_encoding='utf-8'
    #####确定encoding类型
    try:
        content.decode('utf-8').encode('utf-8')
        source_encoding='utf-8'
    except:
        try:
            content.decode('gbk').encode('utf-8')
            source_encoding='gbk'
        except:
            try:
                content.decode('gb2312').encode('utf-8')
                source_encoding='gb2312'
            except:
                try:
                    content.decode('gb18030').encode('utf-8')
                    source_encoding='gb18030'
                except:
                    try:
                        content.decode('big5').encode('utf-8')
                        source_encoding='gb18030'
                    except:
                        content.decode('cp936').encode('utf-8')
                        source_encoding='cp936'
    f.close()
    
    #####按照确定的encoding读取文件内容，并另存为utf-8编码：
    block_size=4096
    with codecs.open(original_file,'r',source_encoding) as f:
        with codecs.open(newfile,'w','utf-8') as f2:
            while True:
                content=f.read(block_size)
                if not content:
                    break
                f2.write(content)

定义一个函数 handleEncoding(original_file,newfile)对每个文件尝试多种编码，包括 utf-8、gbk、gb2312、gb18030、cp936。按照确定的encoding读取文件内容，并另存为utf-8编码。

for i in range(4):
    handleEncoding(oldfile_names[i],newfile_names[i])

调用 handleEncoding()函数。

import pandas as pd
pd.set_option('display.max_rows',10)

user = pd.read_csv('data_用户表_new.csv',engine='python',encoding='utf-8')
month2 = pd.read_csv('data_2月用户行为数据_new.csv',engine='python',encoding='utf-8')
month3 = pd.read_csv('data_3月用户行为数据_new.csv',engine='python',encoding='utf-8')
month4 = pd.read_csv('data_4月用户行为数据_new.csv',engine='python',encoding='utf-8')

最后顺利读取。

代码007普通

打赏 收藏 海报 链接