WhatTheMike/Fox-V3
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

add save_every for less disk intensive work.

Browse Source
This commit is contained in:
bobloy 2021-03-25 10:01:56 -04:00
parent b4f20dd7d2
commit 59fd96fc5a
1 changed files with 11 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
chatter/trainers.py
View File

@ -142,6 +142,8 @@ class MovieTrainer(KaggleTrainer):
# # lines_dict = {row[0].strip('"'): row[4] for row in reader_list}
statements_from_file = []
save_every = 50
count = 0
# [characterID of first, characterID of second, movieID, list of utterances]
async for lines in AsyncIter(conv):
@ -167,9 +169,15 @@ class MovieTrainer(KaggleTrainer):
statements_from_file.append(statement)
if statements_from_file:
self.chatbot.storage.create_many(statements_from_file)
statements_from_file = []
count += 1
if count >= save_every:
if statements_from_file:
self.chatbot.storage.create_many(statements_from_file)
statements_from_file = []
count = 0
if statements_from_file:
self.chatbot.storage.create_many(statements_from_file)
log.info("Training took", time.time() - start_time, "seconds.")