62 lines
2.1 KiB
Python
62 lines
2.1 KiB
Python
|
import re
|
|||
|
|
|
|||
|
|
from unstructured.nlp.patterns import (
|
|||
|
|
DOUBLE_PARAGRAPH_PATTERN_RE,
|
|||
|
|
E_BULLET_PATTERN,
|
|||
|
|
PARAGRAPH_PATTERN,
|
|||
|
|
PARAGRAPH_PATTERN_RE,
|
|||
|
|
UNICODE_BULLETS_RE,
|
|||
|
|
)
|
|||
|
|
from unstructured.cleaners.core import group_bullet_paragraph
|
|||
|
|
|
|||
|
|
def custom_group_broken_paragraphs(
|
|||
|
|
text: str,
|
|||
|
|
line_split: re.Pattern = PARAGRAPH_PATTERN_RE,
|
|||
|
|
paragraph_split: re.Pattern = DOUBLE_PARAGRAPH_PATTERN_RE,
|
|||
|
|
) -> str:
|
|||
|
|
"""Groups paragraphs that have line breaks for visual/formatting purposes.
|
|||
|
|
For example:
|
|||
|
|
|
|||
|
|
'''The big red fox
|
|||
|
|
is walking down the lane.
|
|||
|
|
|
|||
|
|
At the end of the lane
|
|||
|
|
the fox met a bear.'''
|
|||
|
|
|
|||
|
|
Gets converted to
|
|||
|
|
|
|||
|
|
'''The big red fox is walking down the lane.
|
|||
|
|
At the end of the land the fox met a bear.'''
|
|||
|
|
"""
|
|||
|
|
paragraphs = paragraph_split.split(text)
|
|||
|
|
clean_paragraphs = []
|
|||
|
|
for paragraph in paragraphs:
|
|||
|
|
if not paragraph.strip():
|
|||
|
|
continue
|
|||
|
|
# NOTE(robinson) - This block is to account for lines like the following that shouldn't be
|
|||
|
|
# grouped together, but aren't separated by a double line break.
|
|||
|
|
# Apache License
|
|||
|
|
# Version 2.0, January 2004
|
|||
|
|
# http://www.apache.org/licenses/
|
|||
|
|
|
|||
|
|
#para_split = line_split.split(paragraph)
|
|||
|
|
|
|||
|
|
# pytesseract converts some bullet points to standalone "e" characters
|
|||
|
|
if UNICODE_BULLETS_RE.match(paragraph.strip()) or E_BULLET_PATTERN.match(paragraph.strip()):
|
|||
|
|
tempList = group_bullet_paragraph(paragraph)
|
|||
|
|
clean_paragraphs.extend(tempList)
|
|||
|
|
#print(f"new 11111:{tempList}")
|
|||
|
|
else:
|
|||
|
|
tempList = re.sub(PARAGRAPH_PATTERN, " ", paragraph)
|
|||
|
|
clean_paragraphs.append(tempList)
|
|||
|
|
#print(f"new 333333:{tempList}")
|
|||
|
|
|
|||
|
|
return "\n\n".join(clean_paragraphs)
|
|||
|
|
|
|||
|
|
|
|||
|
|
# str1 = "手工分段**绝缘装置（10） 工作斗在额定载荷下起升至最大平台高度，制动后15 min, 工作斗下沉量应不超过该工况最大 平台高度的0.3%。"
|
|||
|
|
# str2 = "手工分段**操控系统（12） 电气系统的要求如下："
|
|||
|
|
# custom_group_broken_paragraphs(str1)
|
|||
|
|
# custom_group_broken_paragraphs(str2)
|
|||
|
|
|