1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29
| 1. 2.doubleWords = {} 3.def train02(filename): 4. doubleWords_pre={} 5. f=open(filename,encoding='utf-8') 6. 7. num=0 8. for line in f.readlines(): 9. 10. line1 = re.findall('[\u4e00-\u9fa5]+', line) 11. for words in line1: 12. 13. words1='B'+words+'E' 14. pre=' ' 15. 16. for word in words1: 17. if pre != ' ': 18. if pre in doubleWords_pre: 19. if word in doubleWords_pre[pre]: 20. doubleWords_pre[pre][word]+=1 21. else: 22. doubleWords_pre[pre][word]=1 23. else: 24. doubleWords_pre[pre]={} 25. pre=word 26. num+=1 27. f.close() 28. 29. np.save("data/doubleWords_pre", doubleWords_pre)
|