import requests
import re
import os
import glob
import openpyxl
headers = {
'Origin': 'http://屏蔽ov.cn',
'Accept-Encoding': 'gzip, deflate',
'Accept-Language': 'zh-CN,zh;q=0.9,zh-TW;q=0.8,en;q=0.7,ja;q=0.6,id;q=0.5,so;q=0.4,und;q=0.3,la;q=0.2,da;q=0.1',
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.120 Safari/537.36',
'Content-Type': 'application/x-www-form-urlencoded',
'Accept': 'application/json, text/javascript, */*',
'Referer': 'http://jtzhzf.屏蔽/WxLaw/Index',
'X-Requested-With': 'XMLHttpRequest',
'Connection': 'keep-alive',
'DNT': '1',
}
def create_dir(dirname):
if not os.path.exists(dirname):
os.mkdir(dirname)
queries_dict = {
'综合类': {
'page': '1',
'LawType': '\u7EFC\u5408\u7C7B',
'total': '5',
'sort': 'ID',
'order': 'desc',
'rows': '20'
},
'道路运政': {
'page': '1',
'LawType': '\u9053\u8DEF\u8FD0\u653F',
'total': '17',
'sort': 'ID',
'order': 'desc',
'rows': '20'
},
'公路路政': {
'page': '1',
'LawType': '\u516C\u8DEF\u8DEF\u653F',
'total': '8',
'sort': 'ID',
'order': 'desc',
'rows': '20'
},
'水路运政': {
'page': '1',
'LawType': '\u6C34\u8DEF\u8FD0\u653F',
'total': '13',
'sort': 'ID',
'order': 'desc',
'rows': '20'
},
'港口行政': {
'page': '1',
'LawType': '\u6E2F\u53E3\u884C\u653F',
'total': '11',
'sort': 'ID',
'order': 'desc',
'rows': '20'
},
'航道行政': {
'page': '1',
'LawType': '\u822A\u9053\u884C\u653F',
'total': '10',
'sort': 'ID',
'order': 'desc',
'rows': '20'
}
}
def save_all_files():
create_dir('datas')
for category, data in queries_dict.items():
response = requests.post('http://jtzhzf.给屏蔽掉/WxLaw/Datagrid', headers=headers, data=data, verify=False)
rjson = response.json()
for row in rjson['rows']:
law_name = row['LawName']
law_type = row['LawType']
article = row['Article']
filename = 'datas/' + law_type + '-' + law_name + '.txt'
with open(filename, 'w') as f:
f.write(article)
def clean_article(article):
article = article.replace(r' ', '')
return re.sub("<.*?>", " ", article)
def num_to_char(num):
chps = '十 一 二 三 四 五 六 七 八 九'.split()
chos_dict = {}
for i in range(10):
chos_dict[i] = chps[i]
if num == 10:
s = '十'
elif num % 10 == 0 and num < 100:
s = chos_dict[int(str(num)[0])] + '十'
elif num < 10:
s = chos_dict[num]
elif num < 20:
last = num % 10
s = '十' + chos_dict[last]
elif num < 100:
num_str = str(num)
first = chos_dict[int(num_str[0])]
last = chos_dict[num % 10]
s = first + '十' + last
elif num == 100:
s = '一百'
else:
last = num % 10
s = '一百零' + chos_dict[last]
return s
def get_chaps_from_text(text):
chaps = []
num = 1
while True:
chp = "第{}章".format(num_to_char(num)
if chp in text:
data, text = text.split(chp, 1)
if data:
chaps.append(remove_space(data))
else:
chaps.append(remove_space(text))
break
num += 1
return chaps
def remove_space(text):
return text.strip().replace(' ', '').replace(' ', '')
def parse_chap(text, item_count):
"""处理一个章节,划分小段"""
items = []
while True:
item = "第{}条".format(num_to_char(item_count))
if item in text:
data, text = text.split(item, 1)
if data:
items.append(remove_space(data))
else:
items.append(remove_space(text))
break
item_count += 1
return item_count, items
def parse_item(text):
"""处理 (一) 这种情况"""
subitems = []
count = 1
while True:
subitem = "({})".format(num_to_char(count))
if subitem in text:
data, text = text.split(subitem, 1)
if data:
subitems.append(remove_space(data))
else:
subitems.append(remove_space(text))
break
count += 1
return subitems
def remove_attach_file(text):
if '附件' in text:
text = text.split('附件')[0]
return text
def parse_article(article, ws):
"""格式化法律内容"""
text = clean_article(article)
if '目录' in text:
text = text.split('第一章', 1)[-1].split('第一章', 1)[-1]
else:
text = text.split('第一章', 1)[-1]
text = '第一章' + text
chaps = get_chaps_from_text(text)
item_count = 1
item_count_flag = 0
insert_row_count = 1
for chap_idx, chap in enumerate(chaps):
current_chap = "第{}章".format(num_to_char(chap_idx + 1))
item_count, items = parse_chap(chap, item_count)
for item_idx, item in enumerate(items):
if item_idx == 0:
current_item = ''
chap_name = item
else:
item_count_flag += 1
current_item = "第{}条".format(num_to_char(item_count_flag))
subitems = parse_item(item)
for subitem_idx, subitem in enumerate(subitems):
if len(subitems) == 1:
current_sub_item = ''
else:
if subitem_idx:
current_sub_item = "({})".format(num_to_char(subitem_idx))
else:
current_sub_item = ''
subitem = remove_attach_file(subitem)
# print(current_chap, current_item, current_sub_item, subitem)
if current_item:
ws.append([insert_row_count, current_chap, ' ', chap_name, current_item, current_sub_item + subitem])
insert_row_count += 1
def get_or_create_workbook(category):
if not os.path.exists(category):
wb = openpyxl.Workbook()
wb.save(filename=category)
else:
wb = openpyxl.load_workbook(category)
return wb
def create_worksheet(wb, sheet_name):
heads = "序号 目录 节数 名称 条目 内容 备注".split()
ws = wb.create_sheet()
ws.title = sheet_name
ws.append(heads)
return ws
def parse_file(filename):
print('parse file: ', filename)
with open(filename) as f:
text = f.read()
# wk, ws
category, sheet_name = filename.replace('.txt', '').split('/')[-1].split('-')
category = category + '.xlsx'
wb = get_or_create_workbook(category)
ws = create_worksheet(wb, sheet_name)
#
parse_article(text, ws)
wb.save(category)
if __name__ == '__main__':
path = 'datas/*.txt'
for i in glob.glob(path):
parse_file(i)