Merge pull request #3381 from westfish/uie_reader_memory_overflow

fix uie dataloader memory overflow

Author: westfish

model_zoo/uie/utils.py

@@ -222,6 +222,10 @@ def reader(data_path, max_seq_len=512):
             if len(content) <= max_content_len:
                 yield json_line
             else:
+                if result['end'] - result['start'] > max_content_len:
+                    logger.warn(
+                        "result['end '] - result ['start'] exceeds max_content_len, which will result in no valid instance being returned"
+                    )
                 result_list = json_line['result_list']
                 json_lines = []
                 accumulate = 0
@@ -230,7 +234,8 @@ def reader(data_path, max_seq_len=512):
 
                     for result in result_list:
                         if result['start'] + 1 <= max_content_len < result[
-                                'end']:
+                                'end'] and result['end'] - result[
+                                    'start'] <= max_content_len:
                             max_content_len = result['start']
                             break