import jsonimport jsonimport json

def combine_jsonl(file1, file2, output_file):
    combined_data = []

    # 读取第一个jsonl文件
    with open(file1, 'r', encoding='utf-8') as f1:
        for line in f1:
            combined_data.append(json.loads(line))

    # 读取第二个jsonl文件
    with open(file2, 'r', encoding='utf-8') as f2:
        for line in f2:
            combined_data.append(json.loads(line))

    # 将合并的数据写入新的jsonl文件
    with open(output_file, 'w', encoding='utf-8') as out:
        for entry in combined_data:
            out.write(json.dumps(entry, ensure_ascii=False) + '\n')

# 示例使用
combine_jsonl('4.jsonl', 'C:\\Users\\LENOVO\\Documents\\WeChat Files\\wxid_3kv9umjbvkr822\\FileStorage\\File\\2024-10\\3mbpp_test1.jsonl', '5.jsonl')

def combine_jsonl(file1, file2, output_file):
    combined_data = []

    # 读取第一个jsonl文件
    with open(file1, 'r', encoding='utf-8') as f1:
        for line in f1:
            combined_data.append(json.loads(line))

    # 读取第二个jsonl文件
    with open(file2, 'r', encoding='utf-8') as f2:
        for line in f2:
            combined_data.append(json.loads(line))

    # 将合并的数据写入新的jsonl文件
    with open(output_file, 'w', encoding='utf-8') as out:
        for entry in combined_data:
            out.write(json.dumps(entry, ensure_ascii=False) + '\n')

# 示例使用
combine_jsonl('4.jsonl', 'C:\\Users\\LENOVO\\Documents\\WeChat Files\\wxid_3kv9umjbvkr822\\FileStorage\\File\\2024-10\\3mbpp_test1.jsonl', '5.jsonl')


def combine_jsonl(file1, file2, output_file):
    combined_data = []

    # 读取第一个jsonl文件
    with open(file1, 'r', encoding='utf-8') as f1:
        for line in f1:
            combined_data.append(json.loads(line))

    # 读取第二个jsonl文件
    with open(file2, 'r', encoding='utf-8') as f2:
        for line in f2:
            combined_data.append(json.loads(line))

    # 将合并的数据写入新的jsonl文件
    with open(output_file, 'w', encoding='utf-8') as out:
        for entry in combined_data:
            out.write(json.dumps(entry, ensure_ascii=False) + '\n')

# 示例使用
combine_jsonl('4.jsonl', 'C:\\Users\\LENOVO\\Documents\\WeChat Files\\wxid_3kv9umjbvkr822\\FileStorage\\File\\2024-10\\3mbpp_test1.jsonl', '5.jsonl')

import pandas as pd


def parquet_to_jsonl(parquet_file, jsonl_file):
    # 使用pandas读取parquet文件
    df = pd.read_parquet(parquet_file, engine='pyarrow')

    # 将DataFrame转换为JSON Lines格式并写入文件
    with open(jsonl_file, 'w') as f:
        for index, row in df.iterrows():
            json_str = row.to_json()
            f.write(json_str + '\n')


# 使用示例
parquet_file = 'E:\\train-00000-of-00001.parquet'
jsonl_file = 'e1.jsonl'
parquet_to_jsonl(parquet_file, jsonl_file)
#print(1)

(line)
        if "output" in data:
            value = data["output"]
            # 查找特定字符串的位置
            index = value.find("\n```python")
            if index!= -1:
                # 如果找到特定字符串，保留其前面的内容
                new_value = value[:index]
                data["input"] = new_value
        f_out.write(json.dumps(data) + '\n')

# 定义两个文件的名称
source_filename = '3.jsonl'
target_filename = '1.jsonl'

# 打开源文件和目标文件
with open(source_filename, 'r',encoding='utf-8') as source_file, open(target_filename, 'a+',encoding='utf-8') as target_file:
    # 检查目标文件是否为空
    target_file.seek(0)  # 移动到文件开头
    is_target_empty = target_file.read(1) == ''

    # 如果目标文件不为空，则移动到文件末尾并添加一个换行符
    if not is_target_empty:
        target_file.seek(0, 2)  # 移动到文件末尾
        target_file.write('\n')

    # 读取源文件内容并写入目标文件
    for line in source_file:
        target_file.write(line)

RYin: 01-20 15:08:16
import json

# 输入和输出文件名
input_filename = 'code1.jsonl'
output_filename = 'code2.jsonl'

# 要删除的键
key_to_delete = "system"

# 读取输入文件，处理每个JSON对象，并写入输出文件
with open(input_filename, 'r', encoding='utf-8') as infile, open(output_filename, 'w', encoding='utf-8') as outfile:
    for line in infile:
        data = json.loads(line)
        # 如果键存在，则删除它
        if key_to_delete in data:
            del data[key_to_delete]
        # 将修改后的JSON对象写回到文件
        json.dump(data, outfile)
        outfile.write('\n')

RYin: 01-20 15:08:25
import json

# 设置文件路径
input_file_path = 'code2.jsonl'
output_file_path = 'code1.jsonl'
target_string = '```python\n'

# 处理 JSONL 文件
with open(input_file_path, 'r', encoding='utf-8') as infile, open(output_file_path, 'w', encoding='utf-8') as outfile:
    for line in infile:
        # 解析每一行 JSON
        data = json.loads(line)

        # 检查目标键是否存在
        if 'output' in data:
            # 删除特定字符串及之前的内容
            content = data['output']
            if target_string in content:
                # 找到目标字符串的位置，并截取
                content = content.split(target_string, 1)[-1]
                data['output'] = content

        # 将修改后的数据写入新的 JSONL 文件
        outfile.write(json.dumps(data) + '\n')

print("处理完成！")

import json

# 设置文件路径
input_file_path = 'code2.jsonl'
output_file_path = 'code1.jsonl'
target_string = '```python\n'

# 处理 JSONL 文件
with open(input_file_path, 'r', encoding='utf-8') as infile, open(output_file_path, 'w', encoding='utf-8') as outfile:
    for line in infile:
        # 解析每一行 JSON
        data = json.loads(line)

        # 检查目标键是否存在
        if 'output' in data:
            # 删除特定字符串及之前的内容
            content = data['output']
            if target_string in content:
                # 找到目标字符串的位置，并截取
                content = content.split(target_string, 1)[-1]
                data['output'] = content

        # 将修改后的数据写入新的 JSONL 文件
        outfile.write(json.dumps(data) + '\n')

print("处理完成！")

import json

# 读取原始jsonl文件并修改内容
with open('C:\\Users\\LENOVO\\Downloads\\output17.jsonl', 'r', encoding='utf-8') as infile, open('y1.jsonl', 'w', encoding='utf-8') as outfile:
    for line in infile:
        data = json.loads(line)
        # 检查并替换input中的第一个出现的字符串"question"
        if 'input' in data:
            data['input'] = data['input'].replace("question", "Instruction", 1)  # 只替换第一个出现的字符串
        # 写入新的jsonl文件
        outfile.write(json.dumps(data, ensure_ascii=False) + '\n')

import json

# 假设我们要将键 "old_key" 的内容更改为 "new_value"
# old_key = "old_key_name"
old_key = "task_id"
new_value = 0

# 读取JSON Lines文件
with open('5.jsonl', 'r',encoding='utf-8') as file:
    lines = file.readlines()

# 创建一个新的列表来保存修改后的JSON对象
updated_lines = []

# 遍历每一行
for line in lines:
    # 解析JSON对象
    data = json.loads(line)
    new_value = new_value + 1

    # 检查键是否存在，如果存在，则修改它
    if old_key in data:
        # 修改键的内容
        data[old_key] = str(new_value ) # 如果你只是想更新键的值，使用这行
        # data[old_key] = new_value  # 如果你只是想更新旧键的值，使用这行
        # 如果要删除旧键，可以使用 del data[old_key]

    # 将修改后的JSON对象添加回列表
    updated_lines.append(json.dumps(data))

# 将修改后的内容写回到一个新的JSON Lines文件
with open('6.jsonl', 'w', encoding='utf-8') as file:
    file.write('\n'.join(updated_lines))

print("文件已更新。")

import requests
from bs4 import BeautifulSoup
headers = {
    "User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36 SLBrowser/9.0.5.8121 SLBChan/111 SLBVPV/64-bit"
}
response = requests.get("https://hf-mirror.com/datasets/openai/openai_humaneval/viewer/openai_humaneval/test?row=0", headers=headers)
html = response.text
soup = BeautifulSoup(html, "html.parser")
# all_content = soup.select('[class="hljs-ln-code"]')
# all_content = soup.find_all("div",{"class":"htmledit_views"})
all_content = soup.find_all("div",{"class":"line-clamp-2"})
for content in all_content:
   # content_string = content.string
     # if "//" not in content.text:
    print(content.text)
# with open("./data.txt","w",encoding="utf-8") as f:
#     f.write(content.text)