From 762b0fd32005e3a403dcb44bae32a53f2f6d1777 Mon Sep 17 00:00:00 2001 From: bobloy Date: Fri, 25 Sep 2020 12:02:13 -0400 Subject: [PATCH 001/133] WIP Twitter training --- chatter/chat.py | 21 ++++++++++---------- chatter/trainers.py | 48 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 10 deletions(-) create mode 100644 chatter/trainers.py diff --git a/chatter/chat.py b/chatter/chat.py index ad8e37b..607457c 100644 --- a/chatter/chat.py +++ b/chatter/chat.py @@ -15,6 +15,8 @@ from redbot.core.commands import Cog from redbot.core.data_manager import cog_data_path from redbot.core.utils.predicates import MessagePredicate +from chatter.trainers import TwitterCorpusTrainer + log = logging.getLogger("red.fox_v3.chatter") @@ -105,15 +107,7 @@ class Chatter(Cog): return msg.clean_content def new_conversation(msg, sent, out_in, delta): - # if sent is None: - # return False - - # Don't do "too short" processing here. Sometimes people don't respond. - # if len(out_in) < 2: - # return False - - # print(msg.created_at - sent) - + # Should always be positive numbers return msg.created_at - sent >= delta for channel in ctx.guild.text_channels: @@ -158,6 +152,11 @@ class Chatter(Cog): return out + def _train_twitter(self, *args, **kwargs): + trainer = TwitterCorpusTrainer(self.chatbot) + trainer.train(*args, **kwargs) + return True + def _train_ubuntu(self): trainer = UbuntuCorpusTrainer(self.chatbot) trainer.train() @@ -479,7 +478,9 @@ class Chatter(Cog): text = message.clean_content async with channel.typing(): - future = await self.loop.run_in_executor(None, self.chatbot.get_response, text) + # Switched to `generate_response` from `get_result` + # Switch back once better conversation detection is used. + future = await self.loop.run_in_executor(None, self.chatbot.generate_response, text) if future and str(future): await channel.send(str(future)) diff --git a/chatter/trainers.py b/chatter/trainers.py new file mode 100644 index 0000000..e6eedba --- /dev/null +++ b/chatter/trainers.py @@ -0,0 +1,48 @@ +from chatterbot import utils +from chatterbot.conversation import Statement +from chatterbot.trainers import Trainer + + +class TwitterCorpusTrainer(Trainer): + def train(self, *args, **kwargs): + """ + Train the chat bot based on the provided list of + statements that represents a single conversation. + """ + import twint + + c = twint.Config() + c.__dict__.update(kwargs) + twint.run.Search(c) + + + previous_statement_text = None + previous_statement_search_text = '' + + statements_to_create = [] + + for conversation_count, text in enumerate(conversation): + if self.show_training_progress: + utils.print_progress_bar( + 'List Trainer', + conversation_count + 1, len(conversation) + ) + + statement_search_text = self.chatbot.storage.tagger.get_text_index_string(text) + + statement = self.get_preprocessed_statement( + Statement( + text=text, + search_text=statement_search_text, + in_response_to=previous_statement_text, + search_in_response_to=previous_statement_search_text, + conversation='training' + ) + ) + + previous_statement_text = statement.text + previous_statement_search_text = statement_search_text + + statements_to_create.append(statement) + + self.chatbot.storage.create_many(statements_to_create) \ No newline at end of file From 960b66a5b88e6ca162a1df09a21c281538106274 Mon Sep 17 00:00:00 2001 From: BogdanWDK Date: Wed, 30 Sep 2020 06:54:18 +0100 Subject: [PATCH 002/133] Language Support Added Supports Language as [p]tts --- tts/tts.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tts/tts.py b/tts/tts.py index 235d585..8584f5a 100644 --- a/tts/tts.py +++ b/tts/tts.py @@ -28,12 +28,12 @@ class TTS(Cog): return @commands.command(aliases=["t2s", "text2"]) - async def tts(self, ctx: commands.Context, *, text: str): - """ - Send Text to speech messages as an mp3 + async def tts(self, ctx: commands.Context, *, lang: str, text: str): """ + Send Text to speech messages as an mp3 + """ mp3_fp = io.BytesIO() - tts = gTTS(text, lang="en") + tts = gTTS(text, lang=lang) tts.write_to_fp(mp3_fp) mp3_fp.seek(0) await ctx.send(file=discord.File(mp3_fp, "text.mp3")) From 479b23f0f33ff960f985e5c38f840472d782df90 Mon Sep 17 00:00:00 2001 From: bobloy Date: Thu, 1 Oct 2020 09:24:26 -0400 Subject: [PATCH 003/133] Get love image right (when cert is fixed) --- lovecalculator/lovecalculator.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lovecalculator/lovecalculator.py b/lovecalculator/lovecalculator.py index 94b6d49..fba6500 100644 --- a/lovecalculator/lovecalculator.py +++ b/lovecalculator/lovecalculator.py @@ -60,14 +60,14 @@ class LoveCalculator(Cog): else: emoji = "💔" title = f"Dr. Love says that the love percentage for {x} and {y} is: {emoji} {description} {emoji}" - except: + except (TypeError, ValueError): title = "Dr. Love has left a note for you." em = discord.Embed( title=title, description=result_text, color=discord.Color.red(), - url=f"https://www.lovecalculator.com/{result_image}", + url=url ) - + em.set_image(url=f"https://www.lovecalculator.com/{result_image}") await ctx.send(embed=em) From 5752ba605637da4005763e0f55aeab4f229d29af Mon Sep 17 00:00:00 2001 From: bobloy Date: Thu, 1 Oct 2020 09:25:16 -0400 Subject: [PATCH 004/133] Black formatting --- lovecalculator/lovecalculator.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/lovecalculator/lovecalculator.py b/lovecalculator/lovecalculator.py index fba6500..20504bd 100644 --- a/lovecalculator/lovecalculator.py +++ b/lovecalculator/lovecalculator.py @@ -64,10 +64,7 @@ class LoveCalculator(Cog): title = "Dr. Love has left a note for you." em = discord.Embed( - title=title, - description=result_text, - color=discord.Color.red(), - url=url + title=title, description=result_text, color=discord.Color.red(), url=url ) em.set_image(url=f"https://www.lovecalculator.com/{result_image}") await ctx.send(embed=em) From 8e0105355ca20e79bc581ef9890e0df90764689b Mon Sep 17 00:00:00 2001 From: bobloy Date: Thu, 1 Oct 2020 14:33:24 -0400 Subject: [PATCH 005/133] fix ww_stop bug when no game is running --- werewolf/werewolf.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/werewolf/werewolf.py b/werewolf/werewolf.py index bd68a6f..8ea5783 100644 --- a/werewolf/werewolf.py +++ b/werewolf/werewolf.py @@ -285,7 +285,8 @@ class Werewolf(Cog): game = await self._get_game(ctx) game.game_over = True - game.current_action.cancel() + if game.current_action: + game.current_action.cancel() await ctx.maybe_send_embed("Game has been stopped") @commands.guild_only() From 815cfcb0315bb88a099c566d7c5faa6aecc11c71 Mon Sep 17 00:00:00 2001 From: bobloy Date: Fri, 2 Oct 2020 12:01:17 -0400 Subject: [PATCH 006/133] Add member role WIP --- timerole/timerole.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/timerole/timerole.py b/timerole/timerole.py index 7484267..45d147a 100644 --- a/timerole/timerole.py +++ b/timerole/timerole.py @@ -27,10 +27,15 @@ class Timerole(Cog): self.bot = bot self.config = Config.get_conf(self, identifier=9811198108111121, force_registration=True) default_global = {} - default_guild = {"announce": None, "roles": {}} + default_guild = {"announce": None} + default_memberrole = {"had_role": False, "check_again_time": None} self.config.register_global(**default_global) self.config.register_guild(**default_guild) + + self.config.init_custom("MemberRole", 2) + self.config.register_custom("MemberRole", **default_memberrole) + self.updating = asyncio.create_task(self.check_hour()) async def red_delete_data_for_user(self, **kwargs): From 44035b78f7b5e4cb6f31518173af50e33c85e969 Mon Sep 17 00:00:00 2001 From: bobloy Date: Fri, 2 Oct 2020 15:13:08 -0400 Subject: [PATCH 007/133] Timerole rewrite --- timerole/timerole.py | 236 ++++++++++++++++++++++++++++--------------- 1 file changed, 156 insertions(+), 80 deletions(-) diff --git a/timerole/timerole.py b/timerole/timerole.py index 45d147a..ac23bd7 100644 --- a/timerole/timerole.py +++ b/timerole/timerole.py @@ -19,6 +19,15 @@ async def sleep_till_next_hour(): await asyncio.sleep((next_hour - datetime.utcnow()).seconds) +async def announce_to_channel(channel, remove_results, title): + if channel is not None and remove_results: + await channel.send(title) + for page in pagify(remove_results, shorten_by=50): + await channel.send(page) + elif remove_results: # Channel is None, log the results + log.info(remove_results) + + class Timerole(Cog): """Add roles to users based on time on server""" @@ -27,7 +36,7 @@ class Timerole(Cog): self.bot = bot self.config = Config.get_conf(self, identifier=9811198108111121, force_registration=True) default_global = {} - default_guild = {"announce": None} + default_guild = {"announce": None, "reapply": True, "roles": {}} default_memberrole = {"had_role": False, "check_again_time": None} self.config.register_global(**default_global) @@ -89,9 +98,7 @@ class Timerole(Cog): await self.config.guild(guild).roles.set_raw(role.id, value=to_set) await ctx.maybe_send_embed( - "Time Role for {0} set to {1} days and {2} hours until added".format( - role.name, days, hours - ) + f"Time Role for {role.name} set to {days} days and {hours} hours until added" ) @timerole.command() @@ -119,9 +126,7 @@ class Timerole(Cog): await self.config.guild(guild).roles.set_raw(role.id, value=to_set) await ctx.maybe_send_embed( - "Time Role for {0} set to {1} days and {2} hours until removed".format( - role.name, days, hours - ) + f"Time Role for {role.name} set to {days} days and {hours} hours until removed" ) @timerole.command() @@ -130,7 +135,15 @@ class Timerole(Cog): guild = ctx.guild await self.config.guild(guild).announce.set(channel.id) - await ctx.send("Announce channel set to {0}".format(channel.mention)) + await ctx.send(f"Announce channel set to {channel.mention}") + + @timerole.command() + async def reapply(self, ctx: commands.Context): + """Toggle reapplying roles if the member loses it somehow. Defaults to True""" + guild = ctx.guild + current_setting = await self.config.guild(guild).reapply() + await self.config.guild(guild).reapply.set(not current_setting) + await ctx.send(f"Reapplying roles is now set to: {not current_setting}") @timerole.command() async def delrole(self, ctx: commands.Context, role: discord.Role): @@ -138,7 +151,7 @@ class Timerole(Cog): guild = ctx.guild await self.config.guild(guild).roles.set_raw(role.id, value=None) - await ctx.send("{0} will no longer be applied".format(role.name)) + await ctx.send(f"{role.name} will no longer be applied") @timerole.command() async def list(self, ctx: commands.Context): @@ -158,89 +171,152 @@ class Timerole(Cog): str(discord.utils.get(guild.roles, id=int(new_id))) for new_id in r_data["required"] ] - out += "{} | {} days | requires: {}\n".format(str(role), r_data["days"], r_roles) + out += f"{role} | {r_data['days']} days | requires: {r_roles}\n" await ctx.maybe_send_embed(out) async def timerole_update(self): - async for guild in AsyncIter(self.bot.guilds): - addlist = [] - removelist = [] + utcnow = datetime.utcnow() + all_guilds = await self.config.all_guilds() - role_dict = await self.config.guild(guild).roles() - if not any(role_data for role_data in role_dict.values()): # No roles + all_mrs = await self.config.custom("MemberRole").all() + + for guild in self.bot.guilds: + guild_id = str(guild.id) + if guild_id not in all_guilds: continue - async for member in AsyncIter(guild.members): - has_roles = [r.id for r in member.roles] - - add_roles = [ - int(rID) - for rID, r_data in role_dict.items() - if r_data is not None and not r_data["remove"] - ] - remove_roles = [ - int(rID) - for rID, r_data in role_dict.items() - if r_data is not None and r_data["remove"] - ] - - check_add_roles = set(add_roles) - set(has_roles) - check_remove_roles = set(remove_roles) & set(has_roles) - - await self.check_required_and_date( - addlist, check_add_roles, has_roles, member, role_dict - ) - await self.check_required_and_date( - removelist, check_remove_roles, has_roles, member, role_dict - ) + add_results = "" + remove_results = "" + reapply = all_guilds[guild_id]["reapply"] + role_dict = all_guilds[guild_id]["roles"] + if not any(role_data for role_data in role_dict.values()): # No roles + continue + + async for member in AsyncIter(guild.members, steps=100): + addlist = [] + removelist = [] + + for role_id, role_data in role_dict.items(): + mr_dict = all_mrs[str(member.id)][role_id] + + # Stop if they've had the role and reapplying is disabled + if not reapply and mr_dict["had_role"]: + continue + + # Stop if the check_again_time hasn't passed yet + if ( + mr_dict["check_again_time"] is not None + and datetime.fromisoformat(mr_dict["check_again_time"]) >= utcnow + ): + continue + member: discord.Member + has_roles = set(r.id for r in member.roles) + + # Stop if they currently have the role (and mark they had it) + if role_id in has_roles: + if not mr_dict["had_role"]: + await self.config.custom( + "MemberRole", member.id, role_id + ).had_role.set(True) + continue + + # Stop if they don't have all the required roles + if "required" in role_data and not set(role_data["required"]) & has_roles: + # Doesn't have required role + continue + + check_time = member.joined_at + timedelta( + days=role_data["days"], + hours=role_data.get("hours", 0), + ) + + # Check if enough time has passed to get the role and save the check_again_time + if check_time <= utcnow: + await self.config.custom( + "MemberRole", member.id, role_id + ).check_again_time.set(check_time.isoformat()) + continue + + if role_data["remove"]: + removelist.append(role_id) + else: + addlist.append(role_id) + + # Done iterating through roles, now add or remove the roles + addlist = [discord.utils.get(guild.roles, id=role_id) for role_id in addlist] + removelist = [discord.utils.get(guild.roles, id=role_id) for role_id in removelist] + + if addlist: + try: + await member.add_roles(*addlist, reason="Timerole", atomic=False) + except (discord.Forbidden, discord.NotFound) as e: + log.exception("Failed Adding Roles") + add_results += f"{member.display_name} : **(Failed Adding Roles)**\n" + else: + add_results += "\n".join( + f"{member.display_name} : {role.name}" for role in addlist + ) + + if removelist: + try: + await member.remove_roles(*removelist, reason="Timerole", atomic=False) + except (discord.Forbidden, discord.NotFound) as e: + log.exception("Failed Removing Roles") + remove_results += f"{member.display_name} : **(Failed Removing Roles)**\n" + else: + remove_results += "\n".join( + f"{member.display_name} : {role.name}" for role in removelist + ) + + # Done iterating through members, now maybe announce to the guild channel = await self.config.guild(guild).announce() if channel is not None: channel = guild.get_channel(channel) - title = "**These members have received the following roles**\n" - await self.announce_roles(title, addlist, channel, guild, to_add=True) + await announce_to_channel(channel, remove_results, title) title = "**These members have lost the following roles**\n" - await self.announce_roles(title, removelist, channel, guild, to_add=False) - - async def announce_roles(self, title, role_list, channel, guild, to_add: True): - results = "" - async for member, role_id in AsyncIter(role_list): - role = discord.utils.get(guild.roles, id=role_id) - try: - if to_add: - await member.add_roles(role, reason="Timerole") - else: - await member.remove_roles(role, reason="Timerole") - except (discord.Forbidden, discord.NotFound) as e: - results += "{} : {} **(Failed)**\n".format(member.display_name, role.name) - else: - results += "{} : {}\n".format(member.display_name, role.name) - if channel is not None and results: - await channel.send(title) - for page in pagify(results, shorten_by=50): - await channel.send(page) - elif results: # Channel is None, log the results - log.info(results) - - async def check_required_and_date(self, role_list, check_roles, has_roles, member, role_dict): - async for role_id in AsyncIter(check_roles): - # Check for required role - if "required" in role_dict[str(role_id)]: - if not set(role_dict[str(role_id)]["required"]) & set(has_roles): - # Doesn't have required role - continue - - if ( - member.joined_at - + timedelta( - days=role_dict[str(role_id)]["days"], - hours=role_dict[str(role_id)].get("hours", 0), - ) - <= datetime.today() - ): - # Qualifies - role_list.append((member, role_id)) + await announce_to_channel(channel, remove_results, title) + # End + + # async def announce_roles(self, title, role_list, channel, guild, to_add: True): + # results = "" + # async for member, role_id in AsyncIter(role_list): + # role = discord.utils.get(guild.roles, id=role_id) + # try: + # if to_add: + # await member.add_roles(role, reason="Timerole") + # else: + # await member.remove_roles(role, reason="Timerole") + # except (discord.Forbidden, discord.NotFound) as e: + # results += f"{member.display_name} : {role.name} **(Failed)**\n" + # else: + # results += f"{member.display_name} : {role.name}\n" + # if channel is not None and results: + # await channel.send(title) + # for page in pagify(results, shorten_by=50): + # await channel.send(page) + # elif results: # Channel is None, log the results + # log.info(results) + + # async def check_required_and_date(self, role_list, check_roles, has_roles, member, role_dict): + # async for role_id in AsyncIter(check_roles): + # # Check for required role + # if "required" in role_dict[str(role_id)]: + # if not set(role_dict[str(role_id)]["required"]) & set(has_roles): + # # Doesn't have required role + # continue + # + # if ( + # member.joined_at + # + timedelta( + # days=role_dict[str(role_id)]["days"], + # hours=role_dict[str(role_id)].get("hours", 0), + # ) + # <= datetime.utcnow() + # ): + # # Qualifies + # role_list.append((member, role_id)) async def check_hour(self): await sleep_till_next_hour() From b141accbd96ac861eec9eda4fcefa3e69a909b7c Mon Sep 17 00:00:00 2001 From: bobloy Date: Fri, 2 Oct 2020 15:32:55 -0400 Subject: [PATCH 008/133] Timerole rewrite WIP --- timerole/timerole.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/timerole/timerole.py b/timerole/timerole.py index ac23bd7..af6f4b4 100644 --- a/timerole/timerole.py +++ b/timerole/timerole.py @@ -178,11 +178,14 @@ class Timerole(Cog): utcnow = datetime.utcnow() all_guilds = await self.config.all_guilds() - all_mrs = await self.config.custom("MemberRole").all() + # all_mrs = await self.config.custom("MemberRole").all() + + log.debug(f"Begin timerole update") for guild in self.bot.guilds: - guild_id = str(guild.id) + guild_id = guild.id if guild_id not in all_guilds: + log.debug(f"Guild has no configured settings: {guild}") continue add_results = "" @@ -191,6 +194,7 @@ class Timerole(Cog): role_dict = all_guilds[guild_id]["roles"] if not any(role_data for role_data in role_dict.values()): # No roles + log.debug(f"No roles are configured for guild: {guild}") continue async for member in AsyncIter(guild.members, steps=100): @@ -198,7 +202,7 @@ class Timerole(Cog): removelist = [] for role_id, role_data in role_dict.items(): - mr_dict = all_mrs[str(member.id)][role_id] + mr_dict = await self.config.custom("MemberRole", member.id, role_id).all() # Stop if they've had the role and reapplying is disabled if not reapply and mr_dict["had_role"]: @@ -222,7 +226,9 @@ class Timerole(Cog): continue # Stop if they don't have all the required roles - if "required" in role_data and not set(role_data["required"]) & has_roles: + if role_data is None or ( + "required" in role_data and not set(role_data["required"]) & has_roles + ): # Doesn't have required role continue From c63a4923e7c947a795aac332838b31001ee2422c Mon Sep 17 00:00:00 2001 From: bobloy Date: Mon, 5 Oct 2020 11:21:57 -0400 Subject: [PATCH 009/133] Actually do the logic right --- timerole/timerole.py | 61 ++++++++++++++++++++++++++++++++------------ 1 file changed, 45 insertions(+), 16 deletions(-) diff --git a/timerole/timerole.py b/timerole/timerole.py index af6f4b4..c079596 100644 --- a/timerole/timerole.py +++ b/timerole/timerole.py @@ -19,13 +19,13 @@ async def sleep_till_next_hour(): await asyncio.sleep((next_hour - datetime.utcnow()).seconds) -async def announce_to_channel(channel, remove_results, title): - if channel is not None and remove_results: +async def announce_to_channel(channel, results, title): + if channel is not None and results: await channel.send(title) - for page in pagify(remove_results, shorten_by=50): + for page in pagify(results, shorten_by=50): await channel.send(page) - elif remove_results: # Channel is None, log the results - log.info(remove_results) + elif results: # Channel is None, log the results + log.info(results) class Timerole(Cog): @@ -197,6 +197,9 @@ class Timerole(Cog): log.debug(f"No roles are configured for guild: {guild}") continue + # all_mr = await self.config.all_custom("MemberRole") + # log.debug(f"{all_mr=}") + async for member in AsyncIter(guild.members, steps=100): addlist = [] removelist = [] @@ -206,6 +209,7 @@ class Timerole(Cog): # Stop if they've had the role and reapplying is disabled if not reapply and mr_dict["had_role"]: + log.debug(f"{member.display_name} - Not reapplying") continue # Stop if the check_again_time hasn't passed yet @@ -213,6 +217,7 @@ class Timerole(Cog): mr_dict["check_again_time"] is not None and datetime.fromisoformat(mr_dict["check_again_time"]) >= utcnow ): + log.debug(f"{member.display_name} - Not time to check again yet") continue member: discord.Member has_roles = set(r.id for r in member.roles) @@ -223,6 +228,9 @@ class Timerole(Cog): await self.config.custom( "MemberRole", member.id, role_id ).had_role.set(True) + log.debug( + f"{member.display_name} - Already has the role, maybe applying `had_role`" + ) continue # Stop if they don't have all the required roles @@ -230,6 +238,7 @@ class Timerole(Cog): "required" in role_data and not set(role_data["required"]) & has_roles ): # Doesn't have required role + # log.debug(f"{member.display_name} - Missing required roles") continue check_time = member.joined_at + timedelta( @@ -238,10 +247,14 @@ class Timerole(Cog): ) # Check if enough time has passed to get the role and save the check_again_time - if check_time <= utcnow: + if check_time >= utcnow: await self.config.custom( "MemberRole", member.id, role_id ).check_again_time.set(check_time.isoformat()) + log.debug( + f"{member.display_name} - Not enough time has passed to qualify for the role\n" + f"Waiting until {check_time}" + ) continue if role_data["remove"]: @@ -250,39 +263,55 @@ class Timerole(Cog): addlist.append(role_id) # Done iterating through roles, now add or remove the roles - addlist = [discord.utils.get(guild.roles, id=role_id) for role_id in addlist] - removelist = [discord.utils.get(guild.roles, id=role_id) for role_id in removelist] + if not addlist and not removelist: + continue + + log.debug(f"{addlist=}\n{removelist=}") + add_roles = [ + discord.utils.get(guild.roles, id=int(role_id)) for role_id in addlist + ] + remove_roles = [ + discord.utils.get(guild.roles, id=int(role_id)) for role_id in removelist + ] + + if None in add_roles or None in remove_roles: + log.info( + f"Timerole ran into an error with the roles in: {add_roles + remove_roles}" + ) if addlist: try: - await member.add_roles(*addlist, reason="Timerole", atomic=False) + await member.add_roles(*add_roles, reason="Timerole", atomic=False) except (discord.Forbidden, discord.NotFound) as e: log.exception("Failed Adding Roles") add_results += f"{member.display_name} : **(Failed Adding Roles)**\n" else: add_results += "\n".join( - f"{member.display_name} : {role.name}" for role in addlist + f"{member.display_name} : {role.name}" for role in add_roles ) if removelist: try: - await member.remove_roles(*removelist, reason="Timerole", atomic=False) + await member.remove_roles(*remove_roles, reason="Timerole", atomic=False) except (discord.Forbidden, discord.NotFound) as e: log.exception("Failed Removing Roles") remove_results += f"{member.display_name} : **(Failed Removing Roles)**\n" else: remove_results += "\n".join( - f"{member.display_name} : {role.name}" for role in removelist + f"{member.display_name} : {role.name}" for role in remove_roles ) # Done iterating through members, now maybe announce to the guild channel = await self.config.guild(guild).announce() if channel is not None: channel = guild.get_channel(channel) - title = "**These members have received the following roles**\n" - await announce_to_channel(channel, remove_results, title) - title = "**These members have lost the following roles**\n" - await announce_to_channel(channel, remove_results, title) + + if add_results: + title = "**These members have received the following roles**\n" + await announce_to_channel(channel, add_results, title) + if remove_results: + title = "**These members have lost the following roles**\n" + await announce_to_channel(channel, remove_results, title) # End # async def announce_roles(self, title, role_list, channel, guild, to_add: True): From 10767da507f02c0b273e60d9ce10606a3001837a Mon Sep 17 00:00:00 2001 From: bobloy Date: Mon, 5 Oct 2020 13:00:58 -0400 Subject: [PATCH 010/133] Clear the reapply logic if the role is deleted --- timerole/timerole.py | 69 +++++++++++++++++++++++++++----------------- 1 file changed, 43 insertions(+), 26 deletions(-) diff --git a/timerole/timerole.py b/timerole/timerole.py index c079596..fdfe584 100644 --- a/timerole/timerole.py +++ b/timerole/timerole.py @@ -1,6 +1,7 @@ import asyncio import logging from datetime import datetime, timedelta +from typing import Optional import discord from redbot.core import Config, checks, commands @@ -37,13 +38,13 @@ class Timerole(Cog): self.config = Config.get_conf(self, identifier=9811198108111121, force_registration=True) default_global = {} default_guild = {"announce": None, "reapply": True, "roles": {}} - default_memberrole = {"had_role": False, "check_again_time": None} + default_rolemember = {"had_role": False, "check_again_time": None} self.config.register_global(**default_global) self.config.register_guild(**default_guild) - self.config.init_custom("MemberRole", 2) - self.config.register_custom("MemberRole", **default_memberrole) + self.config.init_custom("RoleMember", 2) + self.config.register_custom("RoleMember", **default_rolemember) self.updating = asyncio.create_task(self.check_hour()) @@ -63,10 +64,12 @@ class Timerole(Cog): Useful for troubleshooting the initial setup """ - + pre_run = datetime.utcnow() async with ctx.typing(): await self.timerole_update() await ctx.tick() + after_run = datetime.utcnow() + await ctx.maybe_send_embed(f"Took {after_run-pre_run} seconds") @commands.group() @checks.mod_or_permissions(administrator=True) @@ -130,12 +133,15 @@ class Timerole(Cog): ) @timerole.command() - async def channel(self, ctx: commands.Context, channel: discord.TextChannel): + async def channel(self, ctx: commands.Context, channel: Optional[discord.TextChannel] = None): """Sets the announce channel for role adds""" guild = ctx.guild - - await self.config.guild(guild).announce.set(channel.id) - await ctx.send(f"Announce channel set to {channel.mention}") + if channel is None: + await self.config.guild(guild).announce.clear() + await ctx.maybe_send_embed(f"Announce channel has been cleared") + else: + await self.config.guild(guild).announce.set(channel.id) + await ctx.send(f"Announce channel set to {channel.mention}") @timerole.command() async def reapply(self, ctx: commands.Context): @@ -143,7 +149,7 @@ class Timerole(Cog): guild = ctx.guild current_setting = await self.config.guild(guild).reapply() await self.config.guild(guild).reapply.set(not current_setting) - await ctx.send(f"Reapplying roles is now set to: {not current_setting}") + await ctx.maybe_send_embed(f"Reapplying roles is now set to: {not current_setting}") @timerole.command() async def delrole(self, ctx: commands.Context, role: discord.Role): @@ -151,7 +157,8 @@ class Timerole(Cog): guild = ctx.guild await self.config.guild(guild).roles.set_raw(role.id, value=None) - await ctx.send(f"{role.name} will no longer be applied") + await self.config.custom("RoleMember", role.id).clear() + await ctx.maybe_send_embed(f"{role.name} will no longer be applied") @timerole.command() async def list(self, ctx: commands.Context): @@ -178,7 +185,7 @@ class Timerole(Cog): utcnow = datetime.utcnow() all_guilds = await self.config.all_guilds() - # all_mrs = await self.config.custom("MemberRole").all() + # all_mrs = await self.config.custom("RoleMember").all() log.debug(f"Begin timerole update") @@ -197,15 +204,19 @@ class Timerole(Cog): log.debug(f"No roles are configured for guild: {guild}") continue - # all_mr = await self.config.all_custom("MemberRole") + # all_mr = await self.config.all_custom("RoleMember") # log.debug(f"{all_mr=}") - async for member in AsyncIter(guild.members, steps=100): + async for member in AsyncIter(guild.members, steps=10): addlist = [] removelist = [] for role_id, role_data in role_dict.items(): - mr_dict = await self.config.custom("MemberRole", member.id, role_id).all() + # Skip non-configured roles + if not role_data: + continue + + mr_dict = await self.config.custom("RoleMember", role_id, member.id).all() # Stop if they've had the role and reapplying is disabled if not reapply and mr_dict["had_role"]: @@ -222,23 +233,21 @@ class Timerole(Cog): member: discord.Member has_roles = set(r.id for r in member.roles) - # Stop if they currently have the role (and mark they had it) - if role_id in has_roles: + # Stop if they currently have or don't have the role, and mark had_role + if (role_id in has_roles and not role_data["remove"]) or ( + role_id not in has_roles and role_data["remove"] + ): if not mr_dict["had_role"]: await self.config.custom( - "MemberRole", member.id, role_id + "RoleMember", role_id, member.id ).had_role.set(True) - log.debug( - f"{member.display_name} - Already has the role, maybe applying `had_role`" - ) + log.debug(f"{member.display_name} - applying had_role") continue # Stop if they don't have all the required roles if role_data is None or ( "required" in role_data and not set(role_data["required"]) & has_roles ): - # Doesn't have required role - # log.debug(f"{member.display_name} - Missing required roles") continue check_time = member.joined_at + timedelta( @@ -249,7 +258,7 @@ class Timerole(Cog): # Check if enough time has passed to get the role and save the check_again_time if check_time >= utcnow: await self.config.custom( - "MemberRole", member.id, role_id + "RoleMember", role_id, member.id ).check_again_time.set(check_time.isoformat()) log.debug( f"{member.display_name} - Not enough time has passed to qualify for the role\n" @@ -266,7 +275,7 @@ class Timerole(Cog): if not addlist and not removelist: continue - log.debug(f"{addlist=}\n{removelist=}") + # log.debug(f"{addlist=}\n{removelist=}") add_roles = [ discord.utils.get(guild.roles, id=int(role_id)) for role_id in addlist ] @@ -286,9 +295,13 @@ class Timerole(Cog): log.exception("Failed Adding Roles") add_results += f"{member.display_name} : **(Failed Adding Roles)**\n" else: - add_results += "\n".join( + add_results += " \n".join( f"{member.display_name} : {role.name}" for role in add_roles ) + for role_id in addlist: + await self.config.custom( + "RoleMember", role_id, member.id + ).had_role.set(True) if removelist: try: @@ -297,9 +310,13 @@ class Timerole(Cog): log.exception("Failed Removing Roles") remove_results += f"{member.display_name} : **(Failed Removing Roles)**\n" else: - remove_results += "\n".join( + remove_results += " \n".join( f"{member.display_name} : {role.name}" for role in remove_roles ) + for role_id in removelist: + await self.config.custom( + "RoleMember", role_id, member.id + ).had_role.set(True) # Done iterating through members, now maybe announce to the guild channel = await self.config.guild(guild).announce() From 4c1cd869303e1407e8d651f11aeffade3b680c52 Mon Sep 17 00:00:00 2001 From: bobloy Date: Mon, 5 Oct 2020 13:06:53 -0400 Subject: [PATCH 011/133] Better time keepking doesn't include tick --- timerole/timerole.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/timerole/timerole.py b/timerole/timerole.py index fdfe584..1b56b69 100644 --- a/timerole/timerole.py +++ b/timerole/timerole.py @@ -64,11 +64,12 @@ class Timerole(Cog): Useful for troubleshooting the initial setup """ - pre_run = datetime.utcnow() async with ctx.typing(): + pre_run = datetime.utcnow() await self.timerole_update() + after_run = datetime.utcnow() await ctx.tick() - after_run = datetime.utcnow() + await ctx.maybe_send_embed(f"Took {after_run-pre_run} seconds") @commands.group() @@ -187,7 +188,7 @@ class Timerole(Cog): # all_mrs = await self.config.custom("RoleMember").all() - log.debug(f"Begin timerole update") + # log.debug(f"Begin timerole update") for guild in self.bot.guilds: guild_id = guild.id From a92c373b497402cff00265e98195eacbc3d3f83f Mon Sep 17 00:00:00 2001 From: bobloy Date: Wed, 7 Oct 2020 15:13:53 -0400 Subject: [PATCH 012/133] New gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 7a224ea..3732d09 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ venv/ v-data/ database.sqlite3 /venv3.4/ +/.venv/ From 1e8d1efb57ffcf764cbfb63b583d760e41b22ec3 Mon Sep 17 00:00:00 2001 From: bobloy Date: Fri, 9 Oct 2020 13:54:05 -0400 Subject: [PATCH 013/133] Respect embed color --- lseen/lseen.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lseen/lseen.py b/lseen/lseen.py index 3348b65..0aff69c 100644 --- a/lseen/lseen.py +++ b/lseen/lseen.py @@ -83,7 +83,7 @@ class LastSeen(Cog): # description="{} was last seen at this date and time".format(member.display_name), # timestamp=last_seen) - embed = discord.Embed(timestamp=last_seen) + embed = discord.Embed(timestamp=last_seen, color=self.bot.get_embed_color(ctx)) await ctx.send(embed=embed) @commands.Cog.listener() From 5611f7abe7e58f6998011cb52455affae543f2aa Mon Sep 17 00:00:00 2001 From: bobloy Date: Fri, 9 Oct 2020 13:55:14 -0400 Subject: [PATCH 014/133] Don't commit before testing --- lseen/lseen.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lseen/lseen.py b/lseen/lseen.py index 0aff69c..69bcf87 100644 --- a/lseen/lseen.py +++ b/lseen/lseen.py @@ -83,7 +83,7 @@ class LastSeen(Cog): # description="{} was last seen at this date and time".format(member.display_name), # timestamp=last_seen) - embed = discord.Embed(timestamp=last_seen, color=self.bot.get_embed_color(ctx)) + embed = discord.Embed(timestamp=last_seen, color=await self.bot.get_embed_color(ctx)) await ctx.send(embed=embed) @commands.Cog.listener() From d71e3afb86573fecd7463787a439d3d77a816ba6 Mon Sep 17 00:00:00 2001 From: bobloy Date: Sun, 11 Oct 2020 14:01:12 -0400 Subject: [PATCH 015/133] Fix re-adding roles bug --- timerole/timerole.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/timerole/timerole.py b/timerole/timerole.py index 1b56b69..714bcc8 100644 --- a/timerole/timerole.py +++ b/timerole/timerole.py @@ -235,8 +235,8 @@ class Timerole(Cog): has_roles = set(r.id for r in member.roles) # Stop if they currently have or don't have the role, and mark had_role - if (role_id in has_roles and not role_data["remove"]) or ( - role_id not in has_roles and role_data["remove"] + if (int(role_id) in has_roles and not role_data["remove"]) or ( + int(role_id) not in has_roles and role_data["remove"] ): if not mr_dict["had_role"]: await self.config.custom( From 5ddafff59f62abb3211aaeec8dbd4b6da844b4de Mon Sep 17 00:00:00 2001 From: bobloy Date: Sat, 17 Oct 2020 12:24:43 -0400 Subject: [PATCH 016/133] Add ability to delete autobanked guildbanks --- stealemoji/stealemoji.py | 49 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 48 insertions(+), 1 deletion(-) diff --git a/stealemoji/stealemoji.py b/stealemoji/stealemoji.py index 492ef70..a492527 100644 --- a/stealemoji/stealemoji.py +++ b/stealemoji/stealemoji.py @@ -50,6 +50,7 @@ class StealEmoji(Cog): default_global = { "stolemoji": {}, "guildbanks": [], + "autobanked_guilds": [], "on": False, "notify": 0, "autobank": False, @@ -145,11 +146,54 @@ class StealEmoji(Cog): await ctx.maybe_send_embed("AutoBanking is now " + str(not curr_setting)) + @checks.is_owner() + @commands.guild_only() + @stealemoji.command(name="deleteserver", aliases=["deleteguild"]) + async def se_deleteserver(self, ctx: commands.Context, guild_id=None): + """Delete servers the bot is the owner of. + + Useful for auto-generated guildbanks.""" + if guild_id is None: + guild = ctx.guild + else: + guild = await self.bot.get_guild(guild_id) + + if guild is None: + await ctx.maybe_send_embed("Failed to get guild, cancelling") + return + guild: discord.Guild + await ctx.maybe_send_embed( + f"Will attempt to delete {guild.name} ({guild.id})\n" f"Okay to continue? (yes/no)" + ) + + def check(m): + return m.author == ctx.author and m.channel == ctx.channel + + try: + answer = await self.bot.wait_for("message", timeout=120, check=check) + except asyncio.TimeoutError: + await ctx.send("Timed out, canceling") + return + + if answer.content.upper() not in ["Y", "YES"]: + await ctx.maybe_send_embed("Cancelling") + return + try: + await guild.delete() + except discord.Forbidden: + log.exception("No permission to delete. I'm probably not the guild owner") + await ctx.maybe_send_embed("No permission to delete. I'm probably not the guild owner") + except discord.HTTPException: + log.exception("Unexpected error when deleting guild") + await ctx.maybe_send_embed("Unexpected error when deleting guild") + else: + await self.bot.send_to_owners(f"Guild {guild.name} deleted") + @checks.is_owner() @commands.guild_only() @stealemoji.command(name="bank") async def se_bank(self, ctx): - """Add current server as emoji bank""" + """Add or remove current server as emoji bank""" def check(m): return ( @@ -235,6 +279,9 @@ class StealEmoji(Cog): return async with self.config.guildbanks() as guildbanks: guildbanks.append(guildbank.id) + # Track generated guilds for easier deletion + async with self.config.autobanked_guilds() as autobanked_guilds: + autobanked_guilds.append(guildbank.id) await asyncio.sleep(2) From 26234e3b18a465ded651960a73ed7d15692a53fb Mon Sep 17 00:00:00 2001 From: bobloy Date: Mon, 19 Oct 2020 15:16:49 -0400 Subject: [PATCH 017/133] Alternate dependencies attempt --- chatter/info.json | 3 +- chatter/trainers.py | 85 +++++++++++++++++++++++---------------------- 2 files changed, 45 insertions(+), 43 deletions(-) diff --git a/chatter/info.json b/chatter/info.json index b79e587..df77ee8 100644 --- a/chatter/info.json +++ b/chatter/info.json @@ -17,7 +17,8 @@ "pytz", "https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-2.3.1/en_core_web_sm-2.3.1.tar.gz#egg=en_core_web_sm", "https://github.com/explosion/spacy-models/releases/download/en_core_web_md-2.3.1/en_core_web_md-2.3.1.tar.gz#egg=en_core_web_md", - "spacy>=2.3,<2.4" + "spacy>=2.3,<2.4", + "--no-deps \"chatterbot>=1.1\"" ], "short": "Local Chatbot run on machine learning", "end_user_data_statement": "This cog only stores anonymous conversations data; no End User Data is stored.", diff --git a/chatter/trainers.py b/chatter/trainers.py index e6eedba..42d6288 100644 --- a/chatter/trainers.py +++ b/chatter/trainers.py @@ -4,45 +4,46 @@ from chatterbot.trainers import Trainer class TwitterCorpusTrainer(Trainer): - def train(self, *args, **kwargs): - """ - Train the chat bot based on the provided list of - statements that represents a single conversation. - """ - import twint - - c = twint.Config() - c.__dict__.update(kwargs) - twint.run.Search(c) - - - previous_statement_text = None - previous_statement_search_text = '' - - statements_to_create = [] - - for conversation_count, text in enumerate(conversation): - if self.show_training_progress: - utils.print_progress_bar( - 'List Trainer', - conversation_count + 1, len(conversation) - ) - - statement_search_text = self.chatbot.storage.tagger.get_text_index_string(text) - - statement = self.get_preprocessed_statement( - Statement( - text=text, - search_text=statement_search_text, - in_response_to=previous_statement_text, - search_in_response_to=previous_statement_search_text, - conversation='training' - ) - ) - - previous_statement_text = statement.text - previous_statement_search_text = statement_search_text - - statements_to_create.append(statement) - - self.chatbot.storage.create_many(statements_to_create) \ No newline at end of file + pass + # def train(self, *args, **kwargs): + # """ + # Train the chat bot based on the provided list of + # statements that represents a single conversation. + # """ + # import twint + # + # c = twint.Config() + # c.__dict__.update(kwargs) + # twint.run.Search(c) + # + # + # previous_statement_text = None + # previous_statement_search_text = '' + # + # statements_to_create = [] + # + # for conversation_count, text in enumerate(conversation): + # if self.show_training_progress: + # utils.print_progress_bar( + # 'List Trainer', + # conversation_count + 1, len(conversation) + # ) + # + # statement_search_text = self.chatbot.storage.tagger.get_text_index_string(text) + # + # statement = self.get_preprocessed_statement( + # Statement( + # text=text, + # search_text=statement_search_text, + # in_response_to=previous_statement_text, + # search_in_response_to=previous_statement_search_text, + # conversation='training' + # ) + # ) + # + # previous_statement_text = statement.text + # previous_statement_search_text = statement_search_text + # + # statements_to_create.append(statement) + # + # self.chatbot.storage.create_many(statements_to_create) \ No newline at end of file From a6ebe02233eadd97cedc3191b680d3a3040dd8fe Mon Sep 17 00:00:00 2001 From: bobloy Date: Mon, 19 Oct 2020 16:09:21 -0400 Subject: [PATCH 018/133] Back to basics --- chatter/info.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/chatter/info.json b/chatter/info.json index df77ee8..b79e587 100644 --- a/chatter/info.json +++ b/chatter/info.json @@ -17,8 +17,7 @@ "pytz", "https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-2.3.1/en_core_web_sm-2.3.1.tar.gz#egg=en_core_web_sm", "https://github.com/explosion/spacy-models/releases/download/en_core_web_md-2.3.1/en_core_web_md-2.3.1.tar.gz#egg=en_core_web_md", - "spacy>=2.3,<2.4", - "--no-deps \"chatterbot>=1.1\"" + "spacy>=2.3,<2.4" ], "short": "Local Chatbot run on machine learning", "end_user_data_statement": "This cog only stores anonymous conversations data; no End User Data is stored.", From 46342109604e2824a3bd011dfbd880fe3909e91c Mon Sep 17 00:00:00 2001 From: bobloy Date: Mon, 19 Oct 2020 16:24:39 -0400 Subject: [PATCH 019/133] Add automatic install option --- chatter/README.md | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/chatter/README.md b/chatter/README.md index 8ef6734..c831bb8 100644 --- a/chatter/README.md +++ b/chatter/README.md @@ -59,6 +59,35 @@ Install these on your windows machine before attempting the installation: [Pandoc - Universal Document Converter](https://pandoc.org/installing.html) ## Methods +### Automatic + +This method requires some luck to pull off. + +#### Step 1: Add repo and install cog + +``` +[p]repo add Fox https://github.com/bobloy/Fox-V3 +[p]cog install Fox chatter +``` + +If you get an error at this step, stop and skip to one of the manual methods below. + +#### Step 2: Install additional dependencies + +Assuming the previous commands had no error, you can now use `pipinstall` to add the remaining dependencies. + +NOTE: This method is not the intended use case for `pipinstall` and may stop working in the future. + +``` +[p]pipinstall --no-deps chatterbot>=1.1 +``` + +#### Step 3: Load the cog and get started + +``` +[p]load chatter +``` + ### Windows - Manually #### Step 1: Built-in Downloader From 675e9b82c88cc58813d0da5f7ccdd85acd165c78 Mon Sep 17 00:00:00 2001 From: bobloy Date: Wed, 21 Oct 2020 14:22:40 -0400 Subject: [PATCH 020/133] Update README.md Fix typo in cog name and unnecessary <> --- chatter/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/chatter/README.md b/chatter/README.md index 8ef6734..d92ad2b 100644 --- a/chatter/README.md +++ b/chatter/README.md @@ -95,7 +95,8 @@ pip install --no-deps "chatterbot>=1.1" #### Step 1: Built-in Downloader ``` -[p]cog install Chatter +[p]repo add Fox https://github.com/bobloy/Fox-V3 +[p]cog install Fox chatter ``` #### Step 2: Install Requirements From 6e2c62d897683ded7799a12241ba9ec653daf77f Mon Sep 17 00:00:00 2001 From: bobloy Date: Wed, 21 Oct 2020 14:47:39 -0400 Subject: [PATCH 021/133] Add appropriate checks --- chatter/chat.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/chatter/chat.py b/chatter/chat.py index ef75bb8..d63a664 100644 --- a/chatter/chat.py +++ b/chatter/chat.py @@ -10,7 +10,7 @@ from chatterbot import ChatBot from chatterbot.comparisons import JaccardSimilarity, LevenshteinDistance, SpacySimilarity from chatterbot.response_selection import get_random_response from chatterbot.trainers import ChatterBotCorpusTrainer, ListTrainer, UbuntuCorpusTrainer -from redbot.core import Config, commands +from redbot.core import Config, checks, commands from redbot.core.commands import Cog from redbot.core.data_manager import cog_data_path from redbot.core.utils.predicates import MessagePredicate @@ -191,6 +191,7 @@ class Chatter(Cog): if ctx.invoked_subcommand is None: pass + @checks.admin() @chatter.command(name="channel") async def chatter_channel( self, ctx: commands.Context, channel: Optional[discord.TextChannel] = None @@ -210,6 +211,7 @@ class Chatter(Cog): await self.config.guild(ctx.guild).chatchannel.set(channel.id) await ctx.maybe_send_embed(f"Chat channel is now {channel.mention}") + @checks.is_owner() @chatter.command(name="cleardata") async def chatter_cleardata(self, ctx: commands.Context, confirm: bool = False): """ @@ -242,6 +244,7 @@ class Chatter(Cog): await ctx.tick() + @checks.is_owner() @chatter.command(name="algorithm", aliases=["algo"]) async def chatter_algorithm( self, ctx: commands.Context, algo_number: int, threshold: float = None @@ -275,6 +278,7 @@ class Chatter(Cog): await ctx.tick() + @checks.is_owner() @chatter.command(name="model") async def chatter_model(self, ctx: commands.Context, model_number: int): """ @@ -312,6 +316,7 @@ class Chatter(Cog): f"Model has been switched to {self.tagger_language.ISO_639_1}" ) + @checks.is_owner() @chatter.command(name="minutes") async def minutes(self, ctx: commands.Context, minutes: int): """ @@ -327,6 +332,7 @@ class Chatter(Cog): await ctx.tick() + @checks.is_owner() @chatter.command(name="age") async def age(self, ctx: commands.Context, days: int): """ @@ -341,6 +347,7 @@ class Chatter(Cog): await self.config.guild(ctx.guild).days.set(days) await ctx.tick() + @checks.is_owner() @chatter.command(name="backup") async def backup(self, ctx, backupname): """ @@ -362,6 +369,7 @@ class Chatter(Cog): else: await ctx.maybe_send_embed("Error occurred :(") + @checks.is_owner() @chatter.command(name="trainubuntu") async def chatter_train_ubuntu(self, ctx: commands.Context, confirmation: bool = False): """ @@ -383,6 +391,7 @@ class Chatter(Cog): else: await ctx.send("Error occurred :(") + @checks.is_owner() @chatter.command(name="trainenglish") async def chatter_train_english(self, ctx: commands.Context): """ @@ -396,6 +405,7 @@ class Chatter(Cog): else: await ctx.maybe_send_embed("Error occurred :(") + @checks.is_owner() @chatter.command() async def train(self, ctx: commands.Context, channel: discord.TextChannel): """ From 5fffaf489397a3e0c6ab1ebc4039c2d60998cef0 Mon Sep 17 00:00:00 2001 From: bobloy Date: Fri, 23 Oct 2020 11:41:11 -0400 Subject: [PATCH 022/133] cog_unload, not __unload, whoops. --- planttycoon/planttycoon.py | 2 +- werewolf/werewolf.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/planttycoon/planttycoon.py b/planttycoon/planttycoon.py index 665fc9a..4209b53 100644 --- a/planttycoon/planttycoon.py +++ b/planttycoon/planttycoon.py @@ -793,7 +793,7 @@ class PlantTycoon(commands.Cog): pass await asyncio.sleep(self.defaults["timers"]["notification"] * 60) - def __unload(self): + def cog_unload(self): self.completion_task.cancel() # self.degradation_task.cancel() self.notification_task.cancel() diff --git a/werewolf/werewolf.py b/werewolf/werewolf.py index bd68a6f..dd711ed 100644 --- a/werewolf/werewolf.py +++ b/werewolf/werewolf.py @@ -56,7 +56,7 @@ class Werewolf(Cog): """Nothing to delete""" return - def __unload(self): + def cog_unload(self): log.debug("Unload called") for game in self.games.values(): del game From fc0870af68607b3f3e79b2b202fa0f05b7a20a6a Mon Sep 17 00:00:00 2001 From: bobloy Date: Wed, 28 Oct 2020 13:08:03 -0400 Subject: [PATCH 023/133] Fix multi-triggers, add `checktask` to see more scheduled executions --- fifo/fifo.py | 53 ++++++++++++++++++++++++++++++++++++++++++++++++++-- fifo/task.py | 16 +--------------- 2 files changed, 52 insertions(+), 17 deletions(-) diff --git a/fifo/fifo.py b/fifo/fifo.py index acd01ac..c42e4df 100644 --- a/fifo/fifo.py +++ b/fifo/fifo.py @@ -1,5 +1,6 @@ +import itertools import logging -from datetime import datetime, timedelta, tzinfo +from datetime import datetime, timedelta, tzinfo, MAXYEAR from typing import Optional, Union import discord @@ -10,7 +11,7 @@ from apscheduler.schedulers.base import STATE_PAUSED, STATE_RUNNING from redbot.core import Config, checks, commands from redbot.core.bot import Red from redbot.core.commands import TimedeltaConverter -from redbot.core.utils.chat_formatting import pagify +from redbot.core.utils.chat_formatting import humanize_list, humanize_timedelta, pagify from .datetime_cron_converters import CronConverter, DatetimeConverter, TimezoneConverter from .task import Task @@ -37,6 +38,27 @@ def _disassemble_job_id(job_id: str): return job_id.split("_") +def _get_run_times(job: Job, now: datetime = None): + """ + Computes the scheduled run times between ``next_run_time`` and ``now`` (inclusive). + + Modified to be asynchronous and yielding instead of all-or-nothing + + """ + if not job.next_run_time: + raise StopIteration() + + if now is None: + now = datetime(MAXYEAR, 12, 31, 23, 59, 59, 999999, tzinfo=job.next_run_time.tzinfo) + yield from _get_run_times(job, now) + raise StopIteration() + + next_run_time = job.next_run_time + while next_run_time and next_run_time <= now: + yield next_run_time + next_run_time = job.trigger.get_next_fire_time(next_run_time, now) + + class FIFO(commands.Cog): """ Simple Scheduling Cog @@ -173,6 +195,30 @@ class FIFO(commands.Cog): if ctx.invoked_subcommand is None: pass + @fifo.command(name="checktask", aliases=["checkjob", "check"]) + async def fifo_checktask(self, ctx: commands.Context, task_name: str): + """Returns the next 10 scheduled executions of the task""" + task = Task(task_name, ctx.guild.id, self.config, bot=self.bot) + await task.load_from_config() + + if task.data is None: + await ctx.maybe_send_embed( + f"Task by the name of {task_name} is not found in this guild" + ) + return + + job = await self._get_job(task) + if job is None: + await ctx.maybe_send_embed("No job scheduled for this task") + return + now = datetime.now(job.next_run_time.tzinfo) + + times = [ + humanize_timedelta(timedelta=x - now) + for x in itertools.islice(_get_run_times(job), 10) + ] + await ctx.maybe_send_embed("\n\n".join(times)) + @fifo.command(name="set") async def fifo_set( self, @@ -326,6 +372,8 @@ class FIFO(commands.Cog): for task_name, task_data in all_tasks.items(): out += f"{task_name}: {task_data}\n" + out = humanize_list(out) + if out: if len(out) > 2000: for page in pagify(out): @@ -394,6 +442,7 @@ class FIFO(commands.Cog): return await task.clear_triggers() + await self._remove_job(task) await ctx.tick() @fifo.group(name="addtrigger", aliases=["trigger"]) diff --git a/fifo/task.py b/fifo/task.py index f7dc45a..4c77e8b 100644 --- a/fifo/task.py +++ b/fifo/task.py @@ -39,7 +39,7 @@ def parse_triggers(data: Union[Dict, None]): return None if len(data["triggers"]) > 1: # Multiple triggers - return OrTrigger(get_trigger(t_data) for t_data in data["triggers"]) + return OrTrigger([get_trigger(t_data) for t_data in data["triggers"]]) return get_trigger(data["triggers"][0]) @@ -108,20 +108,6 @@ class Task: "tzinfo": getattr(t["tzinfo"], "zone", None), } ) - # triggers.append( - # { - # "type": t["type"], - # "time_data": { - # "year": dt.year, - # "month": dt.month, - # "day": dt.day, - # "hour": dt.hour, - # "minute": dt.minute, - # "second": dt.second, - # "tzinfo": dt.tzinfo, - # }, - # } - # ) continue if t["type"] == "cron": From db538f75304cd4acf858771d41ff4c83f8d065f9 Mon Sep 17 00:00:00 2001 From: bobloy Date: Wed, 28 Oct 2020 15:19:28 -0400 Subject: [PATCH 024/133] Hotfix cause I didn't test this --- fifo/fifo.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/fifo/fifo.py b/fifo/fifo.py index c42e4df..4a0e622 100644 --- a/fifo/fifo.py +++ b/fifo/fifo.py @@ -370,9 +370,7 @@ class FIFO(commands.Cog): out = "" all_tasks = await self.config.guild(ctx.guild).tasks() for task_name, task_data in all_tasks.items(): - out += f"{task_name}: {task_data}\n" - - out = humanize_list(out) + out += f"{task_name}: {task_data}\n\n" if out: if len(out) > 2000: From 1f1d116a560e302c5bcc3fc0980536cddead696c Mon Sep 17 00:00:00 2001 From: bobloy Date: Fri, 30 Oct 2020 12:17:35 -0400 Subject: [PATCH 025/133] Docstring update --- nudity/nudity.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/nudity/nudity.py b/nudity/nudity.py index 4233460..64ec02a 100644 --- a/nudity/nudity.py +++ b/nudity/nudity.py @@ -8,9 +8,7 @@ from redbot.core.data_manager import cog_data_path class Nudity(commands.Cog): - """ - V3 Cog Template - """ + """Monitor images for NSFW content and moves them to a nsfw channel if possible""" def __init__(self, bot: Red): super().__init__() From 8015e4a46d00b20778bad524775940b930db6148 Mon Sep 17 00:00:00 2001 From: bobloy Date: Mon, 2 Nov 2020 11:49:42 -0500 Subject: [PATCH 026/133] Hotfix scheduling snowflake issue --- fifo/fifo.py | 1 + fifo/task.py | 13 +++++++++---- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/fifo/fifo.py b/fifo/fifo.py index 4a0e622..c6479b4 100644 --- a/fifo/fifo.py +++ b/fifo/fifo.py @@ -27,6 +27,7 @@ async def _execute_task(task_state): task = Task(**task_state) if await task.load_from_config(): return await task.execute() + log.warning(f"Failed to load data on {task_state=}") return False diff --git a/fifo/task.py b/fifo/task.py index 4c77e8b..9e0e545 100644 --- a/fifo/task.py +++ b/fifo/task.py @@ -253,15 +253,15 @@ class Task: actual_message = await author.history(limit=1).flatten() if not actual_message: # Okay, the *author* has never sent a message? log.warning("No message found in channel cache yet, skipping execution") - return + return False actual_message = actual_message[0] message = FakeMessage(actual_message) # message = FakeMessage2 message.author = author - message.guild = guild # Just in case we got desperate + message.guild = guild # Just in case we got desperate, see above message.channel = channel - message.id = time_snowflake(datetime.now()) # Pretend to be now + message.id = time_snowflake(datetime.utcnow(), high=False) # Pretend to be now message = neuter_message(message) # absolutely weird that this takes a message object instead of guild @@ -273,7 +273,12 @@ class Task: message.content = f"{prefix}{self.get_command_str()}" - if not message.guild or not message.author or not message.content: + if ( + not message.guild + or not message.author + or not message.content + or message.content == prefix + ): log.warning(f"Could not execute task due to message problem: {message}") return False From 419863b07a89ae1d3b8963fee82adb835b4e0993 Mon Sep 17 00:00:00 2001 From: bobloy Date: Wed, 4 Nov 2020 09:04:32 -0500 Subject: [PATCH 027/133] Better error logging. --- fifo/task.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/fifo/task.py b/fifo/task.py index 9e0e545..7c51ee4 100644 --- a/fifo/task.py +++ b/fifo/task.py @@ -225,20 +225,26 @@ class Task: async def execute(self): if not self.data or not self.get_command_str(): - log.warning(f"Could not execute task due to data problem: {self.data=}") + log.warning(f"Could not execute Task[{self.name}] due to data problem: {self.data=}") return False guild: discord.Guild = self.bot.get_guild(self.guild_id) # used for get_prefix if guild is None: - log.warning(f"Could not execute task due to missing guild: {self.guild_id}") + log.warning( + f"Could not execute Task[{self.name}] due to missing guild: {self.guild_id}" + ) return False channel: discord.TextChannel = guild.get_channel(self.channel_id) if channel is None: - log.warning(f"Could not execute task due to missing channel: {self.channel_id}") + log.warning( + f"Could not execute Task[{self.name}] due to missing channel: {self.channel_id}" + ) return False author: discord.User = guild.get_member(self.author_id) if author is None: - log.warning(f"Could not execute task due to missing author: {self.author_id}") + log.warning( + f"Could not execute Task[{self.name}] due to missing author: {self.author_id}" + ) return False actual_message: discord.Message = channel.last_message @@ -279,14 +285,15 @@ class Task: or not message.content or message.content == prefix ): - log.warning(f"Could not execute task due to message problem: {message}") + log.warning(f"Could not execute Task[{self.name}] due to message problem: {message}") return False new_ctx: commands.Context = await self.bot.get_context(message) new_ctx.assume_yes = True if not new_ctx.valid: log.warning( - f"Could not execute Task[{self.name}] due invalid context: {new_ctx.invoked_with}" + f"Could not execute Task[{self.name}] due invalid context: " + f"{new_ctx.invoked_with=} {new_ctx.prefix=} {new_ctx.command=}" ) return False From a2948322f9d37f53d9a63edaf1932a926be26537 Mon Sep 17 00:00:00 2001 From: bobloy Date: Fri, 6 Nov 2020 12:42:03 -0500 Subject: [PATCH 028/133] Download ubuntu data to the cog data directory --- chatter/chat.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/chatter/chat.py b/chatter/chat.py index d63a664..4e3400c 100644 --- a/chatter/chat.py +++ b/chatter/chat.py @@ -159,7 +159,9 @@ class Chatter(Cog): return out def _train_ubuntu(self): - trainer = UbuntuCorpusTrainer(self.chatbot) + trainer = UbuntuCorpusTrainer( + self.chatbot, ubuntu_corpus_data_directory=cog_data_path(self) / "ubuntu_data" + ) trainer.train() return True From 99ab9fc1b484bd6f0c18f185b48238a65a3c474a Mon Sep 17 00:00:00 2001 From: bobloy Date: Wed, 11 Nov 2020 09:44:49 -0500 Subject: [PATCH 029/133] [HOTFIX] Fix not applying the similarity threshold --- chatter/chat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chatter/chat.py b/chatter/chat.py index 4e3400c..41affb6 100644 --- a/chatter/chat.py +++ b/chatter/chat.py @@ -272,7 +272,7 @@ class Chatter(Cog): ) return else: - self.similarity_algo = threshold + self.similarity_threshold = threshold self.similarity_algo = algos[algo_number] async with ctx.typing(): From 2ea077bb0cec5a9fe338e49f89377559d74d72cd Mon Sep 17 00:00:00 2001 From: bobloy Date: Wed, 11 Nov 2020 10:38:54 -0500 Subject: [PATCH 030/133] Add relative trigger, better error handling --- fifo/fifo.py | 44 ++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 42 insertions(+), 2 deletions(-) diff --git a/fifo/fifo.py b/fifo/fifo.py index c6479b4..9bfe2e1 100644 --- a/fifo/fifo.py +++ b/fifo/fifo.py @@ -152,10 +152,16 @@ class FIFO(commands.Cog): return job async def _pause_job(self, task: Task): - return self.scheduler.pause_job(job_id=_assemble_job_id(task.name, task.guild_id)) + try: + return self.scheduler.pause_job(job_id=_assemble_job_id(task.name, task.guild_id)) + except JobLookupError: + return False async def _remove_job(self, task: Task): - return self.scheduler.remove_job(job_id=_assemble_job_id(task.name, task.guild_id)) + try: + self.scheduler.remove_job(job_id=_assemble_job_id(task.name, task.guild_id)) + except JobLookupError: + pass async def _get_tz(self, user: Union[discord.User, discord.Member]) -> Union[None, tzinfo]: if self.tz_cog is None: @@ -483,6 +489,40 @@ class FIFO(commands.Cog): f"Next run time: {job.next_run_time} ({delta_from_now.total_seconds()} seconds)" ) + @fifo_trigger.command(name="relative") + async def fifo_trigger_relative( + self, ctx: commands.Context, task_name: str, *, time_from_now: TimedeltaConverter + ): + """ + Add a "run once" trigger at a time relative from now to the specified task + """ + + task = Task(task_name, ctx.guild.id, self.config) + await task.load_from_config() + + if task.data is None: + await ctx.maybe_send_embed( + f"Task by the name of {task_name} is not found in this guild" + ) + return + + time_to_run = datetime.now() + time_from_now + + result = await task.add_trigger("date", time_to_run, time_to_run.tzinfo) + if not result: + await ctx.maybe_send_embed( + "Failed to add a date trigger to this task, see console for logs" + ) + return + + await task.save_data() + job: Job = await self._process_task(task) + delta_from_now: timedelta = job.next_run_time - datetime.now(job.next_run_time.tzinfo) + await ctx.maybe_send_embed( + f"Task `{task_name}` added {time_to_run} to its scheduled runtimes\n" + f"Next run time: {job.next_run_time} ({delta_from_now.total_seconds()} seconds)" + ) + @fifo_trigger.command(name="date") async def fifo_trigger_date( self, ctx: commands.Context, task_name: str, *, datetime_str: DatetimeConverter From 40b01cff262e49aac82d5421bce6ea03e4845dc2 Mon Sep 17 00:00:00 2001 From: bobloy Date: Wed, 11 Nov 2020 10:40:49 -0500 Subject: [PATCH 031/133] Black formatting --- fifo/fifo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fifo/fifo.py b/fifo/fifo.py index 9bfe2e1..f060211 100644 --- a/fifo/fifo.py +++ b/fifo/fifo.py @@ -491,7 +491,7 @@ class FIFO(commands.Cog): @fifo_trigger.command(name="relative") async def fifo_trigger_relative( - self, ctx: commands.Context, task_name: str, *, time_from_now: TimedeltaConverter + self, ctx: commands.Context, task_name: str, *, time_from_now: TimedeltaConverter ): """ Add a "run once" trigger at a time relative from now to the specified task From ca8c762e691b33fc29821fad947c2b5284484c4e Mon Sep 17 00:00:00 2001 From: bobloy Date: Tue, 17 Nov 2020 15:14:50 -0500 Subject: [PATCH 032/133] FIFO resturcture --- fifo/fifo.py | 54 ++++++++-- fifo/redconfigjobstore.py | 218 ++++++++++++++++++++++---------------- fifo/task.py | 4 +- 3 files changed, 176 insertions(+), 100 deletions(-) diff --git a/fifo/fifo.py b/fifo/fifo.py index f060211..b3272b1 100644 --- a/fifo/fifo.py +++ b/fifo/fifo.py @@ -22,8 +22,8 @@ schedule_log.setLevel(logging.DEBUG) log = logging.getLogger("red.fox_v3.fifo") -async def _execute_task(task_state): - log.info(f"Executing {task_state=}") +async def _execute_task(**task_state): + log.info(f"Executing {task_state.get('name')}") task = Task(**task_state) if await task.load_from_config(): return await task.execute() @@ -60,6 +60,19 @@ def _get_run_times(job: Job, now: datetime = None): next_run_time = job.trigger.get_next_fire_time(next_run_time, now) +class CapturePrint: + """Silly little class to get `print` output""" + + def __init__(self): + self.string = None + + def write(self, string): + if self.string is None: + self.string = string + else: + self.string = self.string + "\n" + string + + class FIFO(commands.Cog): """ Simple Scheduling Cog @@ -78,7 +91,7 @@ class FIFO(commands.Cog): self.config.register_global(**default_global) self.config.register_guild(**default_guild) - self.scheduler = None + self.scheduler: Optional[AsyncIOScheduler] = None self.jobstore = None self.tz_cog = None @@ -94,7 +107,12 @@ class FIFO(commands.Cog): async def initialize(self): - job_defaults = {"coalesce": False, "max_instances": 1} + job_defaults = { + "coalesce": True, + "max_instances": 5, + "misfire_grace_time": 15, + "replace_existing": True, + } # executors = {"default": AsyncIOExecutor()} @@ -104,7 +122,7 @@ class FIFO(commands.Cog): from .redconfigjobstore import RedConfigJobStore self.jobstore = RedConfigJobStore(self.config, self.bot) - await self.jobstore.load_from_config(self.scheduler, "default") + await self.jobstore.load_from_config() self.scheduler.add_jobstore(self.jobstore, "default") self.scheduler.start() @@ -139,9 +157,10 @@ class FIFO(commands.Cog): async def _add_job(self, task: Task): return self.scheduler.add_job( _execute_task, - args=[task.__getstate__()], + kwargs=task.__getstate__(), id=_assemble_job_id(task.name, task.guild_id), trigger=await task.get_combined_trigger(), + name=task.name, ) async def _resume_job(self, task: Task): @@ -372,7 +391,7 @@ class FIFO(commands.Cog): Do `[p]fifo list True` to see tasks from all guilds """ if all_guilds: - pass + pass # TODO: All guilds else: out = "" all_tasks = await self.config.guild(ctx.guild).tasks() @@ -388,6 +407,27 @@ class FIFO(commands.Cog): else: await ctx.maybe_send_embed("No tasks to list") + @fifo.command(name="printschedule") + async def fifo_printschedule(self, ctx: commands.Context): + """ + Print the current schedule of execution. + + Useful for debugging. + """ + cp = CapturePrint() + self.scheduler.print_jobs(out=cp) + + out = cp.string + + if out: + if len(out) > 2000: + for page in pagify(out): + await ctx.maybe_send_embed(page) + else: + await ctx.maybe_send_embed(out) + else: + await ctx.maybe_send_embed("Failed to get schedule from scheduler") + @fifo.command(name="add") async def fifo_add(self, ctx: commands.Context, task_name: str, *, command_to_execute: str): """ diff --git a/fifo/redconfigjobstore.py b/fifo/redconfigjobstore.py index 7e68697..27324f6 100644 --- a/fifo/redconfigjobstore.py +++ b/fifo/redconfigjobstore.py @@ -28,7 +28,7 @@ class RedConfigJobStore(MemoryJobStore): self.config = config self.bot = bot self.pickle_protocol = pickle.HIGHEST_PROTOCOL - self._eventloop = self.bot.loop + self._eventloop = self.bot.loop # Used for @run_in_event_loop # TODO: self.config.jobs_index is never used, # fine but maybe a sign of inefficient use of config @@ -40,32 +40,50 @@ class RedConfigJobStore(MemoryJobStore): @run_in_event_loop def start(self, scheduler, alias): super().start(scheduler, alias) + for job, timestamp in self._jobs: + job._scheduler = self._scheduler + job._jobstore_alias = self._alias - async def load_from_config(self, scheduler, alias): - super().start(scheduler, alias) + async def load_from_config(self): _jobs = await self.config.jobs() - self._jobs = [ - (await self._decode_job(job), timestamp) async for (job, timestamp) in AsyncIter(_jobs) - ] + # self._jobs = [ + # (await self._decode_job(job), timestamp) async for (job, timestamp) in AsyncIter(_jobs) + # ] + async for job, timestamp in AsyncIter(_jobs): + job = await self._decode_job(job) + index = self._get_job_index(timestamp, job.id) + self._jobs.insert(index, (job, timestamp)) + self._jobs_index[job.id] = (job, timestamp) + + async def save_to_config(self): + """Yea that's basically it""" + await self.config.jobs.set( + [(self._encode_job(job), timestamp) for job, timestamp in self._jobs] + ) + # self._jobs_index = await self.config.jobs_index.all() # Overwritten by next - self._jobs_index = {job.id: (job, timestamp) for job, timestamp in self._jobs} + # self._jobs_index = {job.id: (job, timestamp) for job, timestamp in self._jobs} def _encode_job(self, job: Job): job_state = job.__getstate__() - new_args = list(job_state["args"]) - new_args[0]["config"] = None - new_args[0]["bot"] = None - job_state["args"] = tuple(new_args) + job_state["kwargs"]["config"] = None + job_state["kwargs"]["bot"] = None + # new_kwargs = job_state["kwargs"] + # new_kwargs["config"] = None + # new_kwargs["bot"] = None + # job_state["kwargs"] = new_kwargs encoded = base64.b64encode(pickle.dumps(job_state, self.pickle_protocol)) out = { "_id": job.id, "next_run_time": datetime_to_utc_timestamp(job.next_run_time), "job_state": encoded.decode("ascii"), } - new_args = list(job_state["args"]) - new_args[0]["config"] = self.config - new_args[0]["bot"] = self.bot - job_state["args"] = tuple(new_args) + job_state["kwargs"]["config"] = self.config + job_state["kwargs"]["bot"] = self.bot + # new_kwargs = job_state["kwargs"] + # new_kwargs["config"] = self.config + # new_kwargs["bot"] = self.bot + # job_state["kwargs"] = new_kwargs # log.debug(f"Encoding job id: {job.id}\n" # f"Encoded as: {out}") @@ -76,10 +94,20 @@ class RedConfigJobStore(MemoryJobStore): return None job_state = in_job["job_state"] job_state = pickle.loads(base64.b64decode(job_state)) - new_args = list(job_state["args"]) - new_args[0]["config"] = self.config - new_args[0]["bot"] = self.bot - job_state["args"] = tuple(new_args) + if job_state["args"]: # Backwards compatibility on args to kwargs + job_state["kwargs"] = { + "name": job_state["args"][0], + "guild_id": job_state["args"][1], + "author_id": job_state["args"][2], + "channel_id": job_state["args"][3], + "bot": job_state["args"][4], + } + job_state["kwargs"]["config"] = self.config + job_state["kwargs"]["bot"] = self.bot + # new_kwargs = job_state["kwargs"] + # new_kwargs["config"] = self.config + # new_kwargs["bot"] = self.bot + # job_state["kwargs"] = new_kwargs job = Job.__new__(Job) job.__setstate__(job_state) job._scheduler = self._scheduler @@ -96,78 +124,82 @@ class RedConfigJobStore(MemoryJobStore): return job - @run_in_event_loop - def add_job(self, job: Job): - if job.id in self._jobs_index: - raise ConflictingIdError(job.id) - # log.debug(f"Check job args: {job.args=}") - timestamp = datetime_to_utc_timestamp(job.next_run_time) - index = self._get_job_index(timestamp, job.id) # This is fine - self._jobs.insert(index, (job, timestamp)) - self._jobs_index[job.id] = (job, timestamp) - asyncio.create_task(self._async_add_job(job, index, timestamp)) - # log.debug(f"Added job: {self._jobs[index][0].args}") - - async def _async_add_job(self, job, index, timestamp): - encoded_job = self._encode_job(job) - job_tuple = tuple([encoded_job, timestamp]) - async with self.config.jobs() as jobs: - jobs.insert(index, job_tuple) - # await self.config.jobs_index.set_raw(job.id, value=job_tuple) - return True - - @run_in_event_loop - def update_job(self, job): - old_tuple: Tuple[Union[Job, None], Union[datetime, None]] = self._jobs_index.get( - job.id, (None, None) - ) - old_job = old_tuple[0] - old_timestamp = old_tuple[1] - if old_job is None: - raise JobLookupError(job.id) - - # If the next run time has not changed, simply replace the job in its present index. - # Otherwise, reinsert the job to the list to preserve the ordering. - old_index = self._get_job_index(old_timestamp, old_job.id) - new_timestamp = datetime_to_utc_timestamp(job.next_run_time) - asyncio.create_task( - self._async_update_job(job, new_timestamp, old_index, old_job, old_timestamp) - ) - - async def _async_update_job(self, job, new_timestamp, old_index, old_job, old_timestamp): - encoded_job = self._encode_job(job) - if old_timestamp == new_timestamp: - self._jobs[old_index] = (job, new_timestamp) - async with self.config.jobs() as jobs: - jobs[old_index] = (encoded_job, new_timestamp) - else: - del self._jobs[old_index] - new_index = self._get_job_index(new_timestamp, job.id) # This is fine - self._jobs.insert(new_index, (job, new_timestamp)) - async with self.config.jobs() as jobs: - del jobs[old_index] - jobs.insert(new_index, (encoded_job, new_timestamp)) - self._jobs_index[old_job.id] = (job, new_timestamp) - # await self.config.jobs_index.set_raw(old_job.id, value=(encoded_job, new_timestamp)) - - log.debug(f"Async Updated {job.id=}") - log.debug(f"Check job args: {job.args=}") - - @run_in_event_loop - def remove_job(self, job_id): - job, timestamp = self._jobs_index.get(job_id, (None, None)) - if job is None: - raise JobLookupError(job_id) - - index = self._get_job_index(timestamp, job_id) - del self._jobs[index] - del self._jobs_index[job.id] - asyncio.create_task(self._async_remove_job(index, job)) - - async def _async_remove_job(self, index, job): - async with self.config.jobs() as jobs: - del jobs[index] - # await self.config.jobs_index.clear_raw(job.id) + # @run_in_event_loop + # def add_job(self, job: Job): + # if job.id in self._jobs_index: + # raise ConflictingIdError(job.id) + # # log.debug(f"Check job args: {job.args=}") + # timestamp = datetime_to_utc_timestamp(job.next_run_time) + # index = self._get_job_index(timestamp, job.id) # This is fine + # self._jobs.insert(index, (job, timestamp)) + # self._jobs_index[job.id] = (job, timestamp) + # task = asyncio.create_task(self._async_add_job(job, index, timestamp)) + # self._eventloop.run_until_complete(task) + # # log.debug(f"Added job: {self._jobs[index][0].args}") + # + # async def _async_add_job(self, job, index, timestamp): + # encoded_job = self._encode_job(job) + # job_tuple = tuple([encoded_job, timestamp]) + # async with self.config.jobs() as jobs: + # jobs.insert(index, job_tuple) + # # await self.config.jobs_index.set_raw(job.id, value=job_tuple) + # return True + + # @run_in_event_loop + # def update_job(self, job): + # old_tuple: Tuple[Union[Job, None], Union[datetime, None]] = self._jobs_index.get( + # job.id, (None, None) + # ) + # old_job = old_tuple[0] + # old_timestamp = old_tuple[1] + # if old_job is None: + # raise JobLookupError(job.id) + # + # # If the next run time has not changed, simply replace the job in its present index. + # # Otherwise, reinsert the job to the list to preserve the ordering. + # old_index = self._get_job_index(old_timestamp, old_job.id) + # new_timestamp = datetime_to_utc_timestamp(job.next_run_time) + # task = asyncio.create_task( + # self._async_update_job(job, new_timestamp, old_index, old_job, old_timestamp) + # ) + # self._eventloop.run_until_complete(task) + # + # async def _async_update_job(self, job, new_timestamp, old_index, old_job, old_timestamp): + # encoded_job = self._encode_job(job) + # if old_timestamp == new_timestamp: + # self._jobs[old_index] = (job, new_timestamp) + # async with self.config.jobs() as jobs: + # jobs[old_index] = (encoded_job, new_timestamp) + # else: + # del self._jobs[old_index] + # new_index = self._get_job_index(new_timestamp, job.id) # This is fine + # self._jobs.insert(new_index, (job, new_timestamp)) + # async with self.config.jobs() as jobs: + # del jobs[old_index] + # jobs.insert(new_index, (encoded_job, new_timestamp)) + # self._jobs_index[old_job.id] = (job, new_timestamp) + # # await self.config.jobs_index.set_raw(old_job.id, value=(encoded_job, new_timestamp)) + # + # log.debug(f"Async Updated {job.id=}") + # # log.debug(f"Check job args: {job.kwargs=}") + + # @run_in_event_loop + # def remove_job(self, job_id): + # """Copied instead of super for the asyncio args""" + # job, timestamp = self._jobs_index.get(job_id, (None, None)) + # if job is None: + # raise JobLookupError(job_id) + # + # index = self._get_job_index(timestamp, job_id) + # del self._jobs[index] + # del self._jobs_index[job.id] + # task = asyncio.create_task(self._async_remove_job(index, job)) + # self._eventloop.run_until_complete(task) + # + # async def _async_remove_job(self, index, job): + # async with self.config.jobs() as jobs: + # del jobs[index] + # # await self.config.jobs_index.clear_raw(job.id) @run_in_event_loop def remove_all_jobs(self): @@ -180,4 +212,8 @@ class RedConfigJobStore(MemoryJobStore): def shutdown(self): """Removes all jobs without clearing config""" + asyncio.create_task(self.async_shutdown()) + + async def async_shutdown(self): + await self.save_to_config() super().remove_all_jobs() diff --git a/fifo/task.py b/fifo/task.py index 7c51ee4..005230b 100644 --- a/fifo/task.py +++ b/fifo/task.py @@ -40,8 +40,8 @@ def parse_triggers(data: Union[Dict, None]): if len(data["triggers"]) > 1: # Multiple triggers return OrTrigger([get_trigger(t_data) for t_data in data["triggers"]]) - - return get_trigger(data["triggers"][0]) + else: + return get_trigger(data["triggers"][0]) class FakeMessage: From a946b1b83b35526d872eb8617d66bbbbcce32a53 Mon Sep 17 00:00:00 2001 From: bobloy Date: Tue, 17 Nov 2020 15:30:07 -0500 Subject: [PATCH 033/133] Catch error better --- fifo/redconfigjobstore.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/fifo/redconfigjobstore.py b/fifo/redconfigjobstore.py index 27324f6..dee020a 100644 --- a/fifo/redconfigjobstore.py +++ b/fifo/redconfigjobstore.py @@ -95,13 +95,16 @@ class RedConfigJobStore(MemoryJobStore): job_state = in_job["job_state"] job_state = pickle.loads(base64.b64decode(job_state)) if job_state["args"]: # Backwards compatibility on args to kwargs - job_state["kwargs"] = { - "name": job_state["args"][0], - "guild_id": job_state["args"][1], - "author_id": job_state["args"][2], - "channel_id": job_state["args"][3], - "bot": job_state["args"][4], - } + try: + job_state["kwargs"] = { + "name": job_state["args"][0], + "guild_id": job_state["args"][1], + "author_id": job_state["args"][2], + "channel_id": job_state["args"][3], + "bot": job_state["args"][4], + } + except IndexError as e: + raise Exception(job_state["args"]) from e job_state["kwargs"]["config"] = self.config job_state["kwargs"]["bot"] = self.bot # new_kwargs = job_state["kwargs"] From 3c3dd2d6cde3d9cbc8c2e25c35490a0b415623c2 Mon Sep 17 00:00:00 2001 From: bobloy Date: Tue, 17 Nov 2020 15:35:17 -0500 Subject: [PATCH 034/133] Correctly handle backwards compatibility --- fifo/redconfigjobstore.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/fifo/redconfigjobstore.py b/fifo/redconfigjobstore.py index dee020a..17caec9 100644 --- a/fifo/redconfigjobstore.py +++ b/fifo/redconfigjobstore.py @@ -95,16 +95,8 @@ class RedConfigJobStore(MemoryJobStore): job_state = in_job["job_state"] job_state = pickle.loads(base64.b64decode(job_state)) if job_state["args"]: # Backwards compatibility on args to kwargs - try: - job_state["kwargs"] = { - "name": job_state["args"][0], - "guild_id": job_state["args"][1], - "author_id": job_state["args"][2], - "channel_id": job_state["args"][3], - "bot": job_state["args"][4], - } - except IndexError as e: - raise Exception(job_state["args"]) from e + job_state["kwargs"] = {**job_state["args"][0]} + job_state["args"] = [] job_state["kwargs"]["config"] = self.config job_state["kwargs"]["bot"] = self.bot # new_kwargs = job_state["kwargs"] From b2ebddc82544584fd57f2d5ffecc72594f980565 Mon Sep 17 00:00:00 2001 From: bobloy Date: Thu, 19 Nov 2020 10:27:22 -0500 Subject: [PATCH 035/133] I forgot to add the bot object for some reason. --- fifo/fifo.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/fifo/fifo.py b/fifo/fifo.py index b3272b1..61eb597 100644 --- a/fifo/fifo.py +++ b/fifo/fifo.py @@ -537,7 +537,7 @@ class FIFO(commands.Cog): Add a "run once" trigger at a time relative from now to the specified task """ - task = Task(task_name, ctx.guild.id, self.config) + task = Task(task_name, ctx.guild.id, self.config, bot=self.bot) await task.load_from_config() if task.data is None: @@ -571,7 +571,7 @@ class FIFO(commands.Cog): Add a "run once" datetime trigger to the specified task """ - task = Task(task_name, ctx.guild.id, self.config) + task = Task(task_name, ctx.guild.id, self.config, bot=self.bot) await task.load_from_config() if task.data is None: @@ -611,7 +611,7 @@ class FIFO(commands.Cog): See https://crontab.guru/ for help generating the cron_str """ - task = Task(task_name, ctx.guild.id, self.config) + task = Task(task_name, ctx.guild.id, self.config, bot=self.bot) await task.load_from_config() if task.data is None: From 19ee6e6f245ba1c1e46022c1896f7274531a7c08 Mon Sep 17 00:00:00 2001 From: bobloy Date: Thu, 19 Nov 2020 10:40:20 -0500 Subject: [PATCH 036/133] Add and remove comments --- fifo/fifo.py | 16 ++++++++-------- fifo/redconfigjobstore.py | 12 +----------- fifo/task.py | 2 +- 3 files changed, 10 insertions(+), 20 deletions(-) diff --git a/fifo/fifo.py b/fifo/fifo.py index 61eb597..d60d777 100644 --- a/fifo/fifo.py +++ b/fifo/fifo.py @@ -1,6 +1,6 @@ import itertools import logging -from datetime import datetime, timedelta, tzinfo, MAXYEAR +from datetime import MAXYEAR, datetime, timedelta, tzinfo from typing import Optional, Union import discord @@ -11,7 +11,7 @@ from apscheduler.schedulers.base import STATE_PAUSED, STATE_RUNNING from redbot.core import Config, checks, commands from redbot.core.bot import Red from redbot.core.commands import TimedeltaConverter -from redbot.core.utils.chat_formatting import humanize_list, humanize_timedelta, pagify +from redbot.core.utils.chat_formatting import humanize_timedelta, pagify from .datetime_cron_converters import CronConverter, DatetimeConverter, TimezoneConverter from .task import Task @@ -108,10 +108,10 @@ class FIFO(commands.Cog): async def initialize(self): job_defaults = { - "coalesce": True, - "max_instances": 5, - "misfire_grace_time": 15, - "replace_existing": True, + "coalesce": True, # Multiple missed triggers within the grace time will only fire once + "max_instances": 5, # This is probably way too high, should likely only be one + "misfire_grace_time": 15, # 15 seconds ain't much, but it's honest work + "replace_existing": True, # Very important for persistent data } # executors = {"default": AsyncIOExecutor()} @@ -119,7 +119,7 @@ class FIFO(commands.Cog): # Default executor is already AsyncIOExecutor self.scheduler = AsyncIOScheduler(job_defaults=job_defaults, logger=schedule_log) - from .redconfigjobstore import RedConfigJobStore + from .redconfigjobstore import RedConfigJobStore # Wait to import to prevent cyclic import self.jobstore = RedConfigJobStore(self.config, self.bot) await self.jobstore.load_from_config() @@ -507,7 +507,7 @@ class FIFO(commands.Cog): """ task = Task(task_name, ctx.guild.id, self.config, bot=self.bot) - await task.load_from_config() + await task.load_from_config() # Will set the channel and author if task.data is None: await ctx.maybe_send_embed( diff --git a/fifo/redconfigjobstore.py b/fifo/redconfigjobstore.py index 17caec9..126cfc0 100644 --- a/fifo/redconfigjobstore.py +++ b/fifo/redconfigjobstore.py @@ -2,17 +2,13 @@ import asyncio import base64 import logging import pickle -from datetime import datetime -from typing import Tuple, Union from apscheduler.job import Job -from apscheduler.jobstores.base import ConflictingIdError, JobLookupError from apscheduler.jobstores.memory import MemoryJobStore from apscheduler.schedulers.asyncio import run_in_event_loop from apscheduler.util import datetime_to_utc_timestamp from redbot.core import Config - -# TODO: use get_lock on config +# TODO: use get_lock on config maybe from redbot.core.bot import Red from redbot.core.utils import AsyncIter @@ -29,13 +25,7 @@ class RedConfigJobStore(MemoryJobStore): self.bot = bot self.pickle_protocol = pickle.HIGHEST_PROTOCOL self._eventloop = self.bot.loop # Used for @run_in_event_loop - # TODO: self.config.jobs_index is never used, - # fine but maybe a sign of inefficient use of config - # task = asyncio.create_task(self.load_from_config()) - # while not task.done(): - # sleep(0.1) - # future = asyncio.ensure_future(self.load_from_config(), loop=self.bot.loop) @run_in_event_loop def start(self, scheduler, alias): diff --git a/fifo/task.py b/fifo/task.py index 005230b..271c59c 100644 --- a/fifo/task.py +++ b/fifo/task.py @@ -166,7 +166,7 @@ class Task: return self.author_id = data["author_id"] - self.guild_id = data["guild_id"] + self.guild_id = data["guild_id"] # Weird I'm doing this, since self.guild_id was just used self.channel_id = data["channel_id"] self.data = data["data"] From 8ab6c5062539faa9423d5ba174a9f2852a822c87 Mon Sep 17 00:00:00 2001 From: bobloy Date: Thu, 19 Nov 2020 13:07:29 -0500 Subject: [PATCH 037/133] HOTFIX: Don't be crzy with pytz timezones --- fifo/timezones.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/fifo/timezones.py b/fifo/timezones.py index 54d7c3e..bd1c239 100644 --- a/fifo/timezones.py +++ b/fifo/timezones.py @@ -5,6 +5,8 @@ All credit to https://github.com/prefrontal/dateutil-parser-timezones """ # from dateutil.tz import gettz +from datetime import datetime + from pytz import timezone @@ -227,4 +229,6 @@ def assemble_timezones(): timezones["YAKT"] = timezone("Asia/Yakutsk") # Yakutsk Time (UTC+09) timezones["YEKT"] = timezone("Asia/Yekaterinburg") # Yekaterinburg Time (UTC+05) + dt = datetime(2020, 1, 1) + timezones.update((x, y.localize(dt).tzinfo) for x, y in timezones.items()) return timezones From 0ff56d933bf0863d7be21ad0d06c19f772d93f0f Mon Sep 17 00:00:00 2001 From: bobloy Date: Wed, 25 Nov 2020 10:05:16 -0500 Subject: [PATCH 038/133] Make relative times better, add fifo wakeup --- fifo/fifo.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/fifo/fifo.py b/fifo/fifo.py index d60d777..9f3bb47 100644 --- a/fifo/fifo.py +++ b/fifo/fifo.py @@ -4,6 +4,7 @@ from datetime import MAXYEAR, datetime, timedelta, tzinfo from typing import Optional, Union import discord +import pytz from apscheduler.job import Job from apscheduler.jobstores.base import JobLookupError from apscheduler.schedulers.asyncio import AsyncIOScheduler @@ -221,6 +222,16 @@ class FIFO(commands.Cog): if ctx.invoked_subcommand is None: pass + @fifo.command(name="wakeup") + async def fifo_wakeup(self, ctx: commands.Context): + """Debug command to fix missed executions. + + If you see a negative "Next run time" when adding a trigger, this may help resolve it. + """ + + self.scheduler.wakeup() + await ctx.tick() + @fifo.command(name="checktask", aliases=["checkjob", "check"]) async def fifo_checktask(self, ctx: commands.Context, task_name: str): """Returns the next 10 scheduled executions of the task""" @@ -546,7 +557,7 @@ class FIFO(commands.Cog): ) return - time_to_run = datetime.now() + time_from_now + time_to_run = datetime.now(pytz.utc) + time_from_now result = await task.add_trigger("date", time_to_run, time_to_run.tzinfo) if not result: From 9411fff5e8cb46f7cf93d19a809ccad73afce4b5 Mon Sep 17 00:00:00 2001 From: bobloy Date: Wed, 25 Nov 2020 10:05:52 -0500 Subject: [PATCH 039/133] Clear old code, shutdown manually, AsyncIter steps --- fifo/redconfigjobstore.py | 82 ++------------------------------------- 1 file changed, 3 insertions(+), 79 deletions(-) diff --git a/fifo/redconfigjobstore.py b/fifo/redconfigjobstore.py index 126cfc0..46528bf 100644 --- a/fifo/redconfigjobstore.py +++ b/fifo/redconfigjobstore.py @@ -39,7 +39,7 @@ class RedConfigJobStore(MemoryJobStore): # self._jobs = [ # (await self._decode_job(job), timestamp) async for (job, timestamp) in AsyncIter(_jobs) # ] - async for job, timestamp in AsyncIter(_jobs): + async for job, timestamp in AsyncIter(_jobs, steps=5): job = await self._decode_job(job) index = self._get_job_index(timestamp, job.id) self._jobs.insert(index, (job, timestamp)) @@ -109,83 +109,6 @@ class RedConfigJobStore(MemoryJobStore): return job - # @run_in_event_loop - # def add_job(self, job: Job): - # if job.id in self._jobs_index: - # raise ConflictingIdError(job.id) - # # log.debug(f"Check job args: {job.args=}") - # timestamp = datetime_to_utc_timestamp(job.next_run_time) - # index = self._get_job_index(timestamp, job.id) # This is fine - # self._jobs.insert(index, (job, timestamp)) - # self._jobs_index[job.id] = (job, timestamp) - # task = asyncio.create_task(self._async_add_job(job, index, timestamp)) - # self._eventloop.run_until_complete(task) - # # log.debug(f"Added job: {self._jobs[index][0].args}") - # - # async def _async_add_job(self, job, index, timestamp): - # encoded_job = self._encode_job(job) - # job_tuple = tuple([encoded_job, timestamp]) - # async with self.config.jobs() as jobs: - # jobs.insert(index, job_tuple) - # # await self.config.jobs_index.set_raw(job.id, value=job_tuple) - # return True - - # @run_in_event_loop - # def update_job(self, job): - # old_tuple: Tuple[Union[Job, None], Union[datetime, None]] = self._jobs_index.get( - # job.id, (None, None) - # ) - # old_job = old_tuple[0] - # old_timestamp = old_tuple[1] - # if old_job is None: - # raise JobLookupError(job.id) - # - # # If the next run time has not changed, simply replace the job in its present index. - # # Otherwise, reinsert the job to the list to preserve the ordering. - # old_index = self._get_job_index(old_timestamp, old_job.id) - # new_timestamp = datetime_to_utc_timestamp(job.next_run_time) - # task = asyncio.create_task( - # self._async_update_job(job, new_timestamp, old_index, old_job, old_timestamp) - # ) - # self._eventloop.run_until_complete(task) - # - # async def _async_update_job(self, job, new_timestamp, old_index, old_job, old_timestamp): - # encoded_job = self._encode_job(job) - # if old_timestamp == new_timestamp: - # self._jobs[old_index] = (job, new_timestamp) - # async with self.config.jobs() as jobs: - # jobs[old_index] = (encoded_job, new_timestamp) - # else: - # del self._jobs[old_index] - # new_index = self._get_job_index(new_timestamp, job.id) # This is fine - # self._jobs.insert(new_index, (job, new_timestamp)) - # async with self.config.jobs() as jobs: - # del jobs[old_index] - # jobs.insert(new_index, (encoded_job, new_timestamp)) - # self._jobs_index[old_job.id] = (job, new_timestamp) - # # await self.config.jobs_index.set_raw(old_job.id, value=(encoded_job, new_timestamp)) - # - # log.debug(f"Async Updated {job.id=}") - # # log.debug(f"Check job args: {job.kwargs=}") - - # @run_in_event_loop - # def remove_job(self, job_id): - # """Copied instead of super for the asyncio args""" - # job, timestamp = self._jobs_index.get(job_id, (None, None)) - # if job is None: - # raise JobLookupError(job_id) - # - # index = self._get_job_index(timestamp, job_id) - # del self._jobs[index] - # del self._jobs_index[job.id] - # task = asyncio.create_task(self._async_remove_job(index, job)) - # self._eventloop.run_until_complete(task) - # - # async def _async_remove_job(self, index, job): - # async with self.config.jobs() as jobs: - # del jobs[index] - # # await self.config.jobs_index.clear_raw(job.id) - @run_in_event_loop def remove_all_jobs(self): super().remove_all_jobs() @@ -201,4 +124,5 @@ class RedConfigJobStore(MemoryJobStore): async def async_shutdown(self): await self.save_to_config() - super().remove_all_jobs() + self._jobs = [] + self._jobs_index = {} From 5ecb8dc826f40f62222c7ccb0b22fe46dcfbcc16 Mon Sep 17 00:00:00 2001 From: bobloy Date: Wed, 25 Nov 2020 10:14:09 -0500 Subject: [PATCH 040/133] Don't schedule jobs without a trigger --- fifo/fifo.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/fifo/fifo.py b/fifo/fifo.py index 9f3bb47..8ff9c80 100644 --- a/fifo/fifo.py +++ b/fifo/fifo.py @@ -148,7 +148,11 @@ class FIFO(commands.Cog): async def _process_task(self, task: Task): job: Union[Job, None] = await self._get_job(task) if job is not None: - job.reschedule(await task.get_combined_trigger()) + combined_trigger_ = await task.get_combined_trigger() + if combined_trigger_ is None: + job.remove() + else: + job.reschedule(combined_trigger_) return job return await self._add_job(task) @@ -156,11 +160,15 @@ class FIFO(commands.Cog): return self.scheduler.get_job(_assemble_job_id(task.name, task.guild_id)) async def _add_job(self, task: Task): + combined_trigger_ = await task.get_combined_trigger() + if combined_trigger_ is None: + return None + return self.scheduler.add_job( _execute_task, kwargs=task.__getstate__(), id=_assemble_job_id(task.name, task.guild_id), - trigger=await task.get_combined_trigger(), + trigger=combined_trigger_, name=task.name, ) From d85f166062ed2c59b8a4f73924645d0d837a920a Mon Sep 17 00:00:00 2001 From: bobloy Date: Wed, 25 Nov 2020 10:14:50 -0500 Subject: [PATCH 041/133] Custom Date that doesn't do past dates --- fifo/date_trigger.py | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 fifo/date_trigger.py diff --git a/fifo/date_trigger.py b/fifo/date_trigger.py new file mode 100644 index 0000000..eb3d617 --- /dev/null +++ b/fifo/date_trigger.py @@ -0,0 +1,7 @@ +from apscheduler.triggers.date import DateTrigger + + +class CustomDateTrigger(DateTrigger): + def get_next_fire_time(self, previous_fire_time, now): + next_run = super().get_next_fire_time(previous_fire_time, now) + return next_run if next_run >= now else None From 51dc2e62d4c4709589e01b78191e4867a9b9ad9e Mon Sep 17 00:00:00 2001 From: bobloy Date: Wed, 25 Nov 2020 10:15:36 -0500 Subject: [PATCH 042/133] Use customdate, check expired before scheduling --- fifo/task.py | 29 +++++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/fifo/task.py b/fifo/task.py index 271c59c..c999ac9 100644 --- a/fifo/task.py +++ b/fifo/task.py @@ -9,10 +9,12 @@ from apscheduler.triggers.cron import CronTrigger from apscheduler.triggers.date import DateTrigger from apscheduler.triggers.interval import IntervalTrigger from discord.utils import time_snowflake -from pytz import timezone +import pytz from redbot.core import Config, commands from redbot.core.bot import Red +from fifo.date_trigger import CustomDateTrigger + log = logging.getLogger("red.fox_v3.fifo.task") @@ -26,7 +28,7 @@ def get_trigger(data): return IntervalTrigger(days=parsed_time.days, seconds=parsed_time.seconds) if data["type"] == "date": - return DateTrigger(data["time_data"], timezone=data["tzinfo"]) + return CustomDateTrigger(data["time_data"], timezone=data["tzinfo"]) if data["type"] == "cron": return CronTrigger.from_crontab(data["time_data"], timezone=data["tzinfo"]) @@ -34,14 +36,25 @@ def get_trigger(data): return False +def check_expired_trigger(trigger: BaseTrigger): + return trigger.get_next_fire_time(None, datetime.now(pytz.utc)) is not None + + def parse_triggers(data: Union[Dict, None]): if data is None or not data.get("triggers", False): # No triggers return None if len(data["triggers"]) > 1: # Multiple triggers - return OrTrigger([get_trigger(t_data) for t_data in data["triggers"]]) + triggers_list = [get_trigger(t_data) for t_data in data["triggers"]] + triggers_list = [t for t in triggers_list if not check_expired_trigger(t)] + if not triggers_list: + return None + return OrTrigger(triggers_list) else: - return get_trigger(data["triggers"][0]) + trigger = get_trigger(data["triggers"][0]) + if check_expired_trigger(trigger): + return None + return trigger class FakeMessage: @@ -66,11 +79,11 @@ def neuter_message(message: FakeMessage): class Task: - default_task_data = {"triggers": [], "command_str": ""} + default_task_data = {"triggers": [], "command_str": "", "expired_triggers": []} default_trigger = { "type": "", - "time_data": None, # Used for Interval and Date Triggers + "time_data": None, "tzinfo": None, } @@ -138,7 +151,7 @@ class Task: # First decode timezone if there is one if t["tzinfo"] is not None: - t["tzinfo"] = timezone(t["tzinfo"]) + t["tzinfo"] = pytz.timezone(t["tzinfo"]) if t["type"] == "interval": # Convert into timedelta t["time_data"] = timedelta(**t["time_data"]) @@ -174,7 +187,7 @@ class Task: await self._decode_time_triggers() return self.data - async def get_triggers(self) -> List[Union[IntervalTrigger, DateTrigger]]: + async def get_triggers(self) -> List[BaseTrigger]: if not self.data: await self.load_from_config() From 624e8863b18ce3f2deac7d268e1616dd63e9ec1f Mon Sep 17 00:00:00 2001 From: bobloy Date: Wed, 25 Nov 2020 14:01:45 -0500 Subject: [PATCH 043/133] Additional expired trigger handling --- fifo/task.py | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/fifo/task.py b/fifo/task.py index c999ac9..281b7d4 100644 --- a/fifo/task.py +++ b/fifo/task.py @@ -100,9 +100,10 @@ class Task: async def _encode_time_triggers(self): if not self.data or not self.data.get("triggers", None): - return [] + return [], [] triggers = [] + expired_triggers = [] for t in self.data["triggers"]: if t["type"] == "interval": # Convert into timedelta td: timedelta = t["time_data"] @@ -114,13 +115,15 @@ class Task: if t["type"] == "date": # Convert into datetime dt: datetime = t["time_data"] - triggers.append( - { - "type": t["type"], - "time_data": dt.isoformat(), - "tzinfo": getattr(t["tzinfo"], "zone", None), - } - ) + data_to_append = { + "type": t["type"], + "time_data": dt.isoformat(), + "tzinfo": getattr(t["tzinfo"], "zone", None), + } + if dt < datetime.now(pytz.utc): + expired_triggers.append(data_to_append) + else: + triggers.append(data_to_append) continue if t["type"] == "cron": @@ -138,7 +141,7 @@ class Task: raise NotImplemented - return triggers + return triggers, expired_triggers async def _decode_time_triggers(self): if not self.data or not self.data.get("triggers", None): @@ -214,7 +217,10 @@ class Task: data_to_save = self.default_task_data.copy() if self.data: data_to_save["command_str"] = self.get_command_str() - data_to_save["triggers"] = await self._encode_time_triggers() + ( + data_to_save["triggers"], + data_to_save["expired_triggers"], + ) = await self._encode_time_triggers() to_save = { "guild_id": self.guild_id, @@ -230,7 +236,10 @@ class Task: return data_to_save = self.data.copy() - data_to_save["triggers"] = await self._encode_time_triggers() + ( + data_to_save["triggers"], + data_to_save["expired_triggers"], + ) = await self._encode_time_triggers() await self.config.guild_from_id(self.guild_id).tasks.set_raw( self.name, "data", value=data_to_save From 9ac89aa369b4f7f30b37ae3b7b74f276e935c89a Mon Sep 17 00:00:00 2001 From: ASSASSIN0831 Date: Wed, 2 Dec 2020 22:09:52 -0500 Subject: [PATCH 044/133] The big update Changes: [FIX]Fixed issued where toggling an infochannel off does not delete the channel [UPDATE] Default counter is now total server members instead of just human users [NEW] Can now toggle off the default counter [NEW] Added a shortcut for infochannelset as icset [NEW] Infochannels are now sorted into a seperate category [NEW ]Added New Counters: Total members(Users+Bots) Roles(Total roles in server) Channels(Total channels in server. Not including infochannels and categorys) Offline Role(members with a specified role) [NEW] Can now customize channel names including the category name --- infochannel/infochannel.py | 675 ++++++++++++++++++++++++++++++++++--- 1 file changed, 631 insertions(+), 44 deletions(-) diff --git a/infochannel/infochannel.py b/infochannel/infochannel.py index b8d36a3..d1a5f4c 100644 --- a/infochannel/infochannel.py +++ b/infochannel/infochannel.py @@ -30,12 +30,33 @@ class InfoChannel(Cog): ) default_guild = { + "category_id": None, "channel_id": None, + "humanchannel_id": None, "botchannel_id": None, + "roleschannel_id": None, + "channels_channel_id": None, "onlinechannel_id": None, + "offlinechannel_id": None, + "role_ids":{}, "member_count": True, + "human_count": False, "bot_count": False, + "roles_count": False, + "channels_count": False, "online_count": False, + "offline_count": False, + "channel_names":{ + "category_name": "Server Stats", + "members_channel": "Total Members: {count}", + "humans_channel": "Humans: {count}", + "bots_channel": "Bots: {count}", + "roles_channel": "Total Roles: {count}", + "channels_channel": "Total Channels: {count}", + "online_channel": "Online: {count}", + "offline_channel": "Offline:{count}", + "role_channel": "{role}: {count}" + } } self.config.register_guild(**default_guild) @@ -61,15 +82,16 @@ class InfoChannel(Cog): ) guild: discord.Guild = ctx.guild - channel_id = await self.config.guild(guild).channel_id() - channel = None - if channel_id is not None: - channel: Union[discord.VoiceChannel, None] = guild.get_channel(channel_id) + category_id = await self.config.guild(guild).category_id() + category = None - if channel_id is not None and channel is None: - await ctx.send("Info channel has been deleted, recreate it?") - elif channel_id is None: - await ctx.send("Enable info channel on this server?") + if category_id is not None: + category: Union[discord.CategoryChannel, None] = guild.get_channel(category_id) + + if category_id is not None and category is None: + await ctx.send("Info category has been deleted, recreate it?") + elif category_id is None: + await ctx.send("Enable info channels on this server?") else: await ctx.send("Do you wish to delete current info channels?") @@ -79,11 +101,11 @@ class InfoChannel(Cog): await ctx.send("Cancelled") return - if channel is None: + if category is None: try: await self.make_infochannel(guild) except discord.Forbidden: - await ctx.send("Failure: Missing permission to create voice channel") + await ctx.send("Failure: Missing permission to create neccessary channels") return else: await self.delete_all_infochannels(guild) @@ -91,7 +113,7 @@ class InfoChannel(Cog): if not await ctx.tick(): await ctx.send("Done!") - @commands.group() + @commands.group(aliases=['icset']) @checks.admin() async def infochannelset(self, ctx: commands.Context): """ @@ -99,7 +121,41 @@ class InfoChannel(Cog): """ if not ctx.invoked_subcommand: pass + + @infochannelset.command(name="membercount") + async def _infochannelset_membercount(self, ctx: commands.Context, enabled: bool = None): + """ + Toggle an infochannel that shows the amount of total members in the server + """ + guild = ctx.guild + if enabled is None: + enabled = not await self.config.guild(guild).member_count() + await self.config.guild(guild).member_count.set(enabled) + await self.make_infochannel(ctx.guild) + + if enabled: + await ctx.send("InfoChannel for member count has been enabled.") + else: + await ctx.send("InfoChannel for member count has been disabled.") + + @infochannelset.command(name="humancount") + async def _infochannelset_humancount(self, ctx: commands.Context, enabled: bool = None): + """ + Toggle an infochannel that shows the amount of human users in the server + """ + guild = ctx.guild + if enabled is None: + enabled = not await self.config.guild(guild).human_count() + + await self.config.guild(guild).human_count.set(enabled) + await self.make_infochannel(ctx.guild) + + if enabled: + await ctx.send("InfoChannel for human user count has been enabled.") + else: + await ctx.send("InfoChannel for human user count has been disabled.") + @infochannelset.command(name="botcount") async def _infochannelset_botcount(self, ctx: commands.Context, enabled: bool = None): """ @@ -117,6 +173,40 @@ class InfoChannel(Cog): else: await ctx.send("InfoChannel for bot count has been disabled.") + @infochannelset.command(name="rolescount") + async def _infochannelset_rolescount(self, ctx: commands.Context, enabled: bool = None): + """ + Toggle an infochannel that shows the amount of roles in the server + """ + guild = ctx.guild + if enabled is None: + enabled = not await self.config.guild(guild).roles_count() + + await self.config.guild(guild).roles_count.set(enabled) + await self.make_infochannel(ctx.guild) + + if enabled: + await ctx.send("InfoChannel for roles count has been enabled.") + else: + await ctx.send("InfoChannel for roles count has been disabled.") + + @infochannelset.command(name="channelscount") + async def _infochannelset_channelscount(self, ctx: commands.Context, enabled: bool = None): + """ + Toggle an infochannel that shows the amount of channels in the server + """ + guild = ctx.guild + if enabled is None: + enabled = not await self.config.guild(guild).channels_count() + + await self.config.guild(guild).channels_count.set(enabled) + await self.make_infochannel(ctx.guild) + + if enabled: + await ctx.send("InfoChannel for channels count has been enabled.") + else: + await ctx.send("InfoChannel for channels count has been disabled.") + @infochannelset.command(name="onlinecount") async def _infochannelset_onlinecount(self, ctx: commands.Context, enabled: bool = None): """ @@ -134,76 +224,477 @@ class InfoChannel(Cog): else: await ctx.send("InfoChannel for online user count has been disabled.") - async def make_infochannel(self, guild: discord.Guild): + @infochannelset.command(name="offlinecount") + async def _infochannelset_offlinecount(self, ctx: commands.Context, enabled: bool = None): + """ + Toggle an infochannel that shows the amount of offline users in the server + """ + guild = ctx.guild + if enabled is None: + enabled = not await self.config.guild(guild).offline_count() + + await self.config.guild(guild).offline_count.set(enabled) + await self.make_infochannel(ctx.guild) + + if enabled: + await ctx.send("InfoChannel for offline user count has been enabled.") + else: + await ctx.send("InfoChannel for offline user count has been disabled.") + + @infochannelset.command(name="rolecount") + async def _infochannelset_rolecount(self, ctx: commands.Context, role: discord.Role, enabled: bool = None): + """ + Toggle an infochannel that shows the amount of users in the server with the specified role + """ + guild = ctx.guild + role_data = await self.config.guild(guild).role_ids.all() + + if str(role.id) in role_data.keys(): + enabled = False + else: + enabled = True + + await self.make_infochannel(ctx.guild, role) + + if enabled: + await ctx.send(f"InfoChannel for {role.name} count has been enabled.") + else: + await ctx.send(f"InfoChannel for {role.name} count has been disabled.") + + #delete later + @infochannelset.command(name="cleardata") + async def _infochannelset_cleardata(self, ctx: commands.Context): + """ + Clears the the servers data in case of corruption + """ + guild = ctx.guild + await self.config.guild(guild).clear() + await ctx.send("The data for this server is cleared.") + + @infochannelset.group(name='name') + async def channelname(self, ctx: commands.Context): + """ + Change the name of the infochannels + """ + if not ctx.invoked_subcommand: + pass + + @channelname.command(name='category') + async def _channelname_Category(self, ctx: commands.Context, *, text): + """ + Change the name of the infochannel's category. + """ + guild = ctx.message.guild + category_id = await self.config.guild(guild).category_id() + category: discord.CategoryChannel = guild.get_channel(category_id) + await category.edit(name = text) + await self.config.guild(guild).channel_names.category_name.set(text) + if not await ctx.tick(): + await ctx.send("Done!") + + @channelname.command(name='members') + async def _channelname_Members(self, ctx: commands.Context, *, text=None): + """ + Change the name of the total members infochannel. + + {count} can be used to display number of total members in the server. + Leave blank to set back to default + Default is set to: + Total Members: {count} + + Example Formats: + Total Members: {count} + {count} Members + """ + guild = ctx.message.guild + if text: + await self.config.guild(guild).channel_names.members_channel.set(text) + else: + await self.config.guild(guild).channel_names.members_channel.clear() + + await self.update_infochannel(guild) + if not await ctx.tick(): + await ctx.send("Done!") + + @channelname.command(name='humans') + async def _channelname_Humans(self, ctx: commands.Context, *, text=None): + """ + Change the name of the human users infochannel. + + {count} can be used to display number of users in the server. + Leave blank to set back to default + Default is set to: + Humans: {count} + + Example Formats: + Users: {count} + {count} Users + """ + guild = ctx.message.guild + if text: + await self.config.guild(guild).channel_names.humans_channel.set(text) + else: + await self.config.guild(guild).channel_names.humans_channel.clear() + + await self.update_infochannel(guild) + if not await ctx.tick(): + await ctx.send("Done!") + + @channelname.command(name='bots') + async def _channelname_Bots(self, ctx: commands.Context, *, text=None): + """ + Change the name of the bots infochannel. + + {count} can be used to display number of bots in the server. + Leave blank to set back to default + Default is set to: + Bots: {count} + + Example Formats: + Total Bots: {count} + {count} Robots + """ + guild = ctx.message.guild + if text: + await self.config.guild(guild).channel_names.bots_channel.set(text) + else: + await self.config.guild(guild).channel_names.bots_channel.clear() + + await self.update_infochannel(guild) + if not await ctx.tick(): + await ctx.send("Done!") + + @channelname.command(name='roles') + async def _channelname_Roles(self, ctx: commands.Context, *, text=None): + """ + Change the name of the roles infochannel. + + Do NOT confuse with the role command that counts number of members with a specified role + + {count} can be used to display number of roles in the server. + Leave blank to set back to default + Default is set to: + Total Roles: {count} + + Example Formats: + Total Roles: {count} + {count} Roles + """ + guild = ctx.message.guild + if text: + await self.config.guild(guild).channel_names.roles_channel.set(text) + else: + await self.config.guild(guild).channel_names.roles_channel.clear() + + await self.update_infochannel(guild) + if not await ctx.tick(): + await ctx.send("Done!") + + @channelname.command(name='channels') + async def _channelname_Channels(self, ctx: commands.Context, *, text=None): + """ + Change the name of the channels infochannel. + + {count} can be used to display number of channels in the server. + This does not count the infochannels + Leave blank to set back to default + Default is set to: + Total Channels: {count} + + Example Formats: + Total Channels: {count} + {count} Channels + """ + guild = ctx.message.guild + if text: + await self.config.guild(guild).channel_names.channels_channel.set(text) + else: + await self.config.guild(guild).channel_names.channels_channel.clear() + + await self.update_infochannel(guild) + if not await ctx.tick(): + await ctx.send("Done!") + + @channelname.command(name='online') + async def _channelname_Online(self, ctx: commands.Context, *, text=None): + """ + Change the name of the online infochannel. + + {count} can be used to display number online members in the server. + Leave blank to set back to default + Default is set to: + Online: {count} + + Example Formats: + Total Online: {count} + {count} Online Members + """ + guild = ctx.message.guild + if text: + await self.config.guild(guild).channel_names.online_channel.set(text) + else: + await self.config.guild(guild).channel_names.online_channel.clear() + + await self.update_infochannel(guild) + if not await ctx.tick(): + await ctx.send("Done!") + + @channelname.command(name='offline') + async def _channelname_Offline(self, ctx: commands.Context, *, text=None): + """ + Change the name of the offline infochannel. + + {count} can be used to display number offline members in the server. + Leave blank to set back to default + Default is set to: + Offline: {count} + + Example Formats: + Total Offline: {count} + {count} Offline Members + """ + guild = ctx.message.guild + if text: + await self.config.guild(guild).channel_names.offline_channel.set(text) + else: + await self.config.guild(guild).channel_names.offline_channel.clear() + + await self.update_infochannel(guild) + if not await ctx.tick(): + await ctx.send("Done!") + + @channelname.command(name='role') + async def _channelname_Role(self, ctx: commands.Context, *, text=None): + """ + Change the name of the infochannel for specific roles. + + All role infochannels follow this format. + Do NOT confuse with the roles command that counts number of roles in the server + + {count} can be used to display number members with the given role. + {role} can be used for the roles name + Leave blank to set back to default + Default is set to: + {role}: {count} + + Example Formats: + {role}: {count} + {count} with {role} role + """ + guild = ctx.message.guild + if text: + await self.config.guild(guild).channel_names.role_channel.set(text) + else: + await self.config.guild(guild).channel_names.role_channel.clear() + + await self.update_infochannel(guild) + if not await ctx.tick(): + await ctx.send("Done!") + + async def make_infochannel(self, guild: discord.Guild, role: discord.Role = None): + membercount = await self.config.guild(guild).member_count() + humancount = await self.config.guild(guild).human_count() botcount = await self.config.guild(guild).bot_count() + rolescount = await self.config.guild(guild).roles_count() + channelscount = await self.config.guild(guild).channels_count() onlinecount = await self.config.guild(guild).online_count() + offlinecount = await self.config.guild(guild).offline_count() overwrites = { guild.default_role: discord.PermissionOverwrite(connect=False), guild.me: discord.PermissionOverwrite(manage_channels=True, connect=True), } - # Remove the old info channel first + # Check for and create the category + category_id = await self.config.guild(guild).category_id() + if category_id is not None: + category: discord.CategoryChannel = guild.get_channel(category_id) + if category is None: + await self.config.guild(guild).category_id.set(None) + category_id = None + + if category_id is None: + category: discord.CategoryChannel = await guild.create_category( + "Server Stats", reason="InfoChannel Category make" + ) + await self.config.guild(guild).category_id.set(category.id) + await category.edit(position = 0) + category_id = category.id + + category: discord.CategoryChannel = guild.get_channel(category_id) + + # Remove the old members channel first channel_id = await self.config.guild(guild).channel_id() - if channel_id is not None: + if category_id is not None: channel: discord.VoiceChannel = guild.get_channel(channel_id) if channel: await channel.delete(reason="InfoChannel delete") - - # Then create the new one - channel = await guild.create_voice_channel( - "Total Humans:", reason="InfoChannel make", overwrites=overwrites - ) - await self.config.guild(guild).channel_id.set(channel.id) - + if membercount: + # Then create the new one + channel = await category.create_voice_channel( + "Total Members:", reason="InfoChannel make", overwrites=overwrites + ) + await self.config.guild(guild).channel_id.set(channel.id) + + # Remove the old human channel first + humanchannel_id = await self.config.guild(guild).humanchannel_id() + if category_id is not None: + humanchannel: discord.VoiceChannel = guild.get_channel(humanchannel_id) + if humanchannel: + await humanchannel.delete(reason="InfoChannel delete") + if humancount: + # Then create the new one + humanchannel = await category.create_voice_channel( + "Humans:", reason="InfoChannel humancount", overwrites=overwrites + ) + await self.config.guild(guild).humanchannel_id.set(humanchannel.id) + + + # Remove the old bot channel first + botchannel_id = await self.config.guild(guild).botchannel_id() + if category_id is not None: + botchannel: discord.VoiceChannel = guild.get_channel(botchannel_id) + if botchannel: + await botchannel.delete(reason="InfoChannel delete") if botcount: - # Remove the old bot channel first - botchannel_id = await self.config.guild(guild).botchannel_id() - if channel_id is not None: - botchannel: discord.VoiceChannel = guild.get_channel(botchannel_id) - if botchannel: - await botchannel.delete(reason="InfoChannel delete") - # Then create the new one - botchannel = await guild.create_voice_channel( + botchannel = await category.create_voice_channel( "Bots:", reason="InfoChannel botcount", overwrites=overwrites ) await self.config.guild(guild).botchannel_id.set(botchannel.id) - if onlinecount: - # Remove the old online channel first - onlinechannel_id = await self.config.guild(guild).onlinechannel_id() - if channel_id is not None: - onlinechannel: discord.VoiceChannel = guild.get_channel(onlinechannel_id) - if onlinechannel: - await onlinechannel.delete(reason="InfoChannel delete") + + # Remove the old roles channel first + roleschannel_id = await self.config.guild(guild).roleschannel_id() + if category_id is not None: + roleschannel: discord.VoiceChannel = guild.get_channel(roleschannel_id) + if roleschannel: + await roleschannel.delete(reason="InfoChannel delete") + + if rolescount: + # Then create the new one + roleschannel = await category.create_voice_channel( + "Total Roles:", reason="InfoChannel rolescount", overwrites=overwrites + ) + await self.config.guild(guild).roleschannel_id.set(roleschannel.id) + + + # Remove the old channels channel first + channels_channel_id = await self.config.guild(guild).channels_channel_id() + if category_id is not None: + channels_channel: discord.VoiceChannel = guild.get_channel(channels_channel_id) + if channels_channel: + await channels_channel.delete(reason="InfoChannel delete") + if channelscount: + # Then create the new one + channels_channel = await category.create_voice_channel( + "Total Channels:", reason="InfoChannel botcount", overwrites=overwrites + ) + await self.config.guild(guild).channels_channel_id.set(channels_channel.id) + + # Remove the old online channel first + onlinechannel_id = await self.config.guild(guild).onlinechannel_id() + if channel_id is not None: + onlinechannel: discord.VoiceChannel = guild.get_channel(onlinechannel_id) + if onlinechannel: + await onlinechannel.delete(reason="InfoChannel delete") + if onlinecount: # Then create the new one - onlinechannel = await guild.create_voice_channel( + onlinechannel = await category.create_voice_channel( "Online:", reason="InfoChannel onlinecount", overwrites=overwrites ) await self.config.guild(guild).onlinechannel_id.set(onlinechannel.id) + + # Remove the old offline channel first + offlinechannel_id = await self.config.guild(guild).offlinechannel_id() + if channel_id is not None: + offlinechannel: discord.VoiceChannel = guild.get_channel(offlinechannel_id) + if offlinechannel: + await offlinechannel.delete(reason="InfoChannel delete") + if offlinecount: + # Then create the new one + offlinechannel = await category.create_voice_channel( + "Offline:", reason="InfoChannel offlinecount", overwrites=overwrites + ) + await self.config.guild(guild).offlinechannel_id.set(offlinechannel.id) + + async with self.config.guild(guild).role_ids() as role_data: + #Remove the old role channels first + for role_id in role_data.keys(): + role_channel_id = role_data[role_id] + if role_channel_id is not None: + rolechannel: discord.VoiceChannel = guild.get_channel(role_channel_id) + if rolechannel: + await rolechannel.delete(reason="InfoChannel delete") + + #The actual toggle for a role counter + if role: + if str(role.id) in role_data.keys(): + role_data.pop(str(role.id)) #if the role is there, then remove it + else: + role_data[role.id] = None #No channel created yet but we want one to be made + if role_data: + # Then create the new ones + for role_id in role_data.keys(): + rolechannel = await category.create_voice_channel( + str(role_id)+":", reason="InfoChannel rolecount", overwrites=overwrites + ) + role_data[role_id] = rolechannel.id await self.update_infochannel(guild) async def delete_all_infochannels(self, guild: discord.Guild): guild_data = await self.config.guild(guild).all() + role_data = guild_data["role_ids"] + category_id = guild_data["category_id"] + humanchannel_id = guild_data["humanchannel_id"] botchannel_id = guild_data["botchannel_id"] + roleschannel_id = guild_data["roleschannel_id"] + channels_channel_id = guild_data["channels_channel_id"] onlinechannel_id = guild_data["onlinechannel_id"] + offlinechannel_id = guild_data["offlinechannel_id"] + category: discord.CategoryChannel = guild.get_channel(category_id) + humanchannel: discord.VoiceChannel = guild.get_channel(humanchannel_id) botchannel: discord.VoiceChannel = guild.get_channel(botchannel_id) + roleschannel: discord.VoiceChannel = guild.get_channel(roleschannel_id) + channels_channel: discord.VoiceChannel = guild.get_channel(channels_channel_id) onlinechannel: discord.VoiceChannel = guild.get_channel(onlinechannel_id) + offlinechannel: discord.VoiceChannel = guild.get_channel(offlinechannel_id) channel_id = guild_data["channel_id"] channel: discord.VoiceChannel = guild.get_channel(channel_id) await channel.delete(reason="InfoChannel delete") + if humanchannel_id is not None: + await humanchannel.delete(reason="InfoChannel delete") if botchannel_id is not None: await botchannel.delete(reason="InfoChannel delete") + if roleschannel_id is not None: + await roleschannel.delete(reason="InfoChannel delete") + if channels_channel is not None: + await channels_channel.delete(reason="InfoChannel delete") if onlinechannel_id is not None: await onlinechannel.delete(reason="InfoChannel delete") - + if offlinechannel_id is not None: + await offlinechannel.delete(reason="InfoChannel delete") + if category_id is not None: + await category.delete(reason="InfoChannel delete") + async with self.config.guild(guild).role_ids() as role_data: + if role_data: + for role_channel_id in role_data.values(): + rolechannel: discord.VoiceChannel = guild.get_channel(role_channel_id) + if rolechannel: + await rolechannel.delete(reason="InfoChannel delete") + await self.config.guild(guild).clear() async def update_infochannel(self, guild: discord.Guild): guild_data = await self.config.guild(guild).all() + humancount = guild_data["human_count"] botcount = guild_data["bot_count"] + rolescount = guild_data["roles_count"] + channelscount = guild_data["channels_count"] onlinecount = guild_data["online_count"] + offlinecount = guild_data["offline_count"] + + category = guild.get_channel(guild_data["category_id"]) # Gets count of bots # bots = lambda x: x.bot @@ -212,40 +703,88 @@ class InfoChannel(Cog): bot_num = len([m for m in guild.members if m.bot]) # bot_msg = f"Bots: {num}" - # Gets count of online users + #Gets count of roles in the server + roles_num = len(guild.roles)-1 + # roles_msg = f"Total Roles: {num}" + + #Gets count of channels in the server + # - - + channels_num = len(guild.channels) - len(category.voice_channels) - len(guild.categories) + # channels_msg = f"Total Channels: {num}" + + # Gets all counts of members members = guild.member_count + # member_msg = f"Total Members: {num}" offline = len(list(filter(lambda m: m.status is discord.Status.offline, guild.members))) + # offline_msg = f"Offline: {num}" online_num = members - offline # online_msg = f"Online: {num}" # Gets count of actual users total = lambda x: not x.bot human_num = len([m for m in guild.members if total(m)]) - # human_msg = f"Total Humans: {num}" + # human_msg = f"Users: {num}" channel_id = guild_data["channel_id"] if channel_id is None: return False botchannel_id = guild_data["botchannel_id"] + roleschannel_id = guild_data["roleschannel_id"] + channels_channel_id = guild_data["channels_channel_id"] onlinechannel_id = guild_data["onlinechannel_id"] + offlinechannel_id = guild_data["offlinechannel_id"] + humanchannel_id = guild_data["humanchannel_id"] channel_id = guild_data["channel_id"] channel: discord.VoiceChannel = guild.get_channel(channel_id) + humanchannel: discord.VoiceChannel = guild.get_channel(humanchannel_id) botchannel: discord.VoiceChannel = guild.get_channel(botchannel_id) + roleschannel: discord.VoiceChannel = guild.get_channel(roleschannel_id) + channels_channel: discord.VoiceChannel = guild.get_channel(channels_channel_id) onlinechannel: discord.VoiceChannel = guild.get_channel(onlinechannel_id) + offlinechannel: discord.VoiceChannel = guild.get_channel(offlinechannel_id) - if guild_data["member_count"]: - name = f"{channel.name.split(':')[0]}: {human_num}" + channel_names = await self.config.guild(guild).channel_names.all() + if guild_data["member_count"]: + name = channel_names["members_channel"].format(count = members) await channel.edit(reason="InfoChannel update", name=name) + if humancount: + name = channel_names["humans_channel"].format(count = human_num) + await humanchannel.edit(reason="InfoChannel update", name=name) + if botcount: - name = f"{botchannel.name.split(':')[0]}: {bot_num}" + name = channel_names["bots_channel"].format(count = bot_num) await botchannel.edit(reason="InfoChannel update", name=name) + + if rolescount: + name = channel_names["roles_channel"].format(count = roles_num) + await roleschannel.edit(reason="InfoChannel update", name=name) + + if channelscount: + name = channel_names["channels_channel"].format(count = channels_num) + await channels_channel.edit(reason="InfoChannel update", name=name) if onlinecount: - name = f"{onlinechannel.name.split(':')[0]}: {online_num}" + name = channel_names["online_channel"].format(count = online_num) await onlinechannel.edit(reason="InfoChannel update", name=name) + + if offlinecount: + name = channel_names["offline_channel"].format(count = offline) + await offlinechannel.edit(reason="InfoChannel update", name=name) + + async with self.config.guild(guild).role_ids() as role_data: + if role_data: + for role_id, role_channel_id in role_data.items(): + rolechannel: discord.VoiceChannel = guild.get_channel(role_channel_id) + role: discord.Role = guild.get_role(int(role_id)) + + role_num = len(role.members) + + name = channel_names["role_channel"].format(count = role_num, role = role.name) + await rolechannel.edit(reason="InfoChannel update", name=name) + async def update_infochannel_with_cooldown(self, guild): """My attempt at preventing rate limits, lets see how it goes""" @@ -291,3 +830,51 @@ class InfoChannel(Cog): if onlinecount: if before.status != after.status: await self.update_infochannel_with_cooldown(after.guild) + role_data = await self.config.guild(after.guild).role_ids.all() + if role_data: + b = set(before.roles) + a = set(after.roles) + if b != a: + await self.update_infochannel_with_cooldown(after.guild) + + @Cog.listener() + async def on_guild_channel_create(self, channel: discord.abc.GuildChannel): + if await self.bot.cog_disabled_in_guild(self, channel.guild): + return + channelscount = await self.config.guild(channel.guild).channels_count() + if channelscount: + await self.update_infochannel_with_cooldown(channel.guild) + + @Cog.listener() + async def on_guild_channel_delete(self, channel: discord.abc.GuildChannel): + if await self.bot.cog_disabled_in_guild(self, channel.guild): + return + channelscount = await self.config.guild(channel.guild).channels_count() + if channelscount: + await self.update_infochannel_with_cooldown(channel.guild) + + @Cog.listener() + async def on_guild_role_create(self, role): + if await self.bot.cog_disabled_in_guild(self, role.guild): + return + + rolescount = await self.config.guild(role.guild).roles_count() + if rolescount: + await self.update_infochannel_with_cooldown(role.guild) + + @Cog.listener() + async def on_guild_role_delete(self, role): + if await self.bot.cog_disabled_in_guild(self, role.guild): + return + + rolescount = await self.config.guild(role.guild).roles_count() + if rolescount: + await self.update_infochannel_with_cooldown(role.guild) + + #delete specific role counter if the role is deleted + async with self.config.guild(role.guild).role_ids() as role_data: + if str(role.id) in role_data.keys(): + role_channel_id = role_data[str(role.id)] + rolechannel: discord.VoiceChannel = role.guild.get_channel(role_channel_id) + await rolechannel.delete(reason="InfoChannel delete") + del role_data[str(role.id)] From bce07f069fc7084ecb51fb06e84db31e28231939 Mon Sep 17 00:00:00 2001 From: ASSASSIN0831 Date: Wed, 2 Dec 2020 22:14:22 -0500 Subject: [PATCH 045/133] Update infochannel.py --- infochannel/infochannel.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/infochannel/infochannel.py b/infochannel/infochannel.py index d1a5f4c..512cf2d 100644 --- a/infochannel/infochannel.py +++ b/infochannel/infochannel.py @@ -261,16 +261,6 @@ class InfoChannel(Cog): else: await ctx.send(f"InfoChannel for {role.name} count has been disabled.") - #delete later - @infochannelset.command(name="cleardata") - async def _infochannelset_cleardata(self, ctx: commands.Context): - """ - Clears the the servers data in case of corruption - """ - guild = ctx.guild - await self.config.guild(guild).clear() - await ctx.send("The data for this server is cleared.") - @infochannelset.group(name='name') async def channelname(self, ctx: commands.Context): """ From 69e2e5acb3d1c34d19e4a9426db9e5faefccaefa Mon Sep 17 00:00:00 2001 From: ASSASSIN0831 Date: Thu, 3 Dec 2020 13:53:17 -0500 Subject: [PATCH 046/133] Black formatting --- infochannel/infochannel.py | 120 ++++++++++++++++++------------------- 1 file changed, 59 insertions(+), 61 deletions(-) diff --git a/infochannel/infochannel.py b/infochannel/infochannel.py index 512cf2d..f021c56 100644 --- a/infochannel/infochannel.py +++ b/infochannel/infochannel.py @@ -38,7 +38,7 @@ class InfoChannel(Cog): "channels_channel_id": None, "onlinechannel_id": None, "offlinechannel_id": None, - "role_ids":{}, + "role_ids": {}, "member_count": True, "human_count": False, "bot_count": False, @@ -46,7 +46,7 @@ class InfoChannel(Cog): "channels_count": False, "online_count": False, "offline_count": False, - "channel_names":{ + "channel_names": { "category_name": "Server Stats", "members_channel": "Total Members: {count}", "humans_channel": "Humans: {count}", @@ -55,8 +55,8 @@ class InfoChannel(Cog): "channels_channel": "Total Channels: {count}", "online_channel": "Online: {count}", "offline_channel": "Offline:{count}", - "role_channel": "{role}: {count}" - } + "role_channel": "{role}: {count}", + }, } self.config.register_guild(**default_guild) @@ -113,7 +113,7 @@ class InfoChannel(Cog): if not await ctx.tick(): await ctx.send("Done!") - @commands.group(aliases=['icset']) + @commands.group(aliases=["icset"]) @checks.admin() async def infochannelset(self, ctx: commands.Context): """ @@ -121,7 +121,7 @@ class InfoChannel(Cog): """ if not ctx.invoked_subcommand: pass - + @infochannelset.command(name="membercount") async def _infochannelset_membercount(self, ctx: commands.Context, enabled: bool = None): """ @@ -155,7 +155,7 @@ class InfoChannel(Cog): await ctx.send("InfoChannel for human user count has been enabled.") else: await ctx.send("InfoChannel for human user count has been disabled.") - + @infochannelset.command(name="botcount") async def _infochannelset_botcount(self, ctx: commands.Context, enabled: bool = None): """ @@ -242,7 +242,9 @@ class InfoChannel(Cog): await ctx.send("InfoChannel for offline user count has been disabled.") @infochannelset.command(name="rolecount") - async def _infochannelset_rolecount(self, ctx: commands.Context, role: discord.Role, enabled: bool = None): + async def _infochannelset_rolecount( + self, ctx: commands.Context, role: discord.Role, enabled: bool = None + ): """ Toggle an infochannel that shows the amount of users in the server with the specified role """ @@ -261,15 +263,15 @@ class InfoChannel(Cog): else: await ctx.send(f"InfoChannel for {role.name} count has been disabled.") - @infochannelset.group(name='name') + @infochannelset.group(name="name") async def channelname(self, ctx: commands.Context): """ Change the name of the infochannels """ if not ctx.invoked_subcommand: pass - - @channelname.command(name='category') + + @channelname.command(name="category") async def _channelname_Category(self, ctx: commands.Context, *, text): """ Change the name of the infochannel's category. @@ -277,12 +279,12 @@ class InfoChannel(Cog): guild = ctx.message.guild category_id = await self.config.guild(guild).category_id() category: discord.CategoryChannel = guild.get_channel(category_id) - await category.edit(name = text) + await category.edit(name=text) await self.config.guild(guild).channel_names.category_name.set(text) if not await ctx.tick(): await ctx.send("Done!") - @channelname.command(name='members') + @channelname.command(name="members") async def _channelname_Members(self, ctx: commands.Context, *, text=None): """ Change the name of the total members infochannel. @@ -306,7 +308,7 @@ class InfoChannel(Cog): if not await ctx.tick(): await ctx.send("Done!") - @channelname.command(name='humans') + @channelname.command(name="humans") async def _channelname_Humans(self, ctx: commands.Context, *, text=None): """ Change the name of the human users infochannel. @@ -329,8 +331,8 @@ class InfoChannel(Cog): await self.update_infochannel(guild) if not await ctx.tick(): await ctx.send("Done!") - - @channelname.command(name='bots') + + @channelname.command(name="bots") async def _channelname_Bots(self, ctx: commands.Context, *, text=None): """ Change the name of the bots infochannel. @@ -354,7 +356,7 @@ class InfoChannel(Cog): if not await ctx.tick(): await ctx.send("Done!") - @channelname.command(name='roles') + @channelname.command(name="roles") async def _channelname_Roles(self, ctx: commands.Context, *, text=None): """ Change the name of the roles infochannel. @@ -379,8 +381,8 @@ class InfoChannel(Cog): await self.update_infochannel(guild) if not await ctx.tick(): await ctx.send("Done!") - - @channelname.command(name='channels') + + @channelname.command(name="channels") async def _channelname_Channels(self, ctx: commands.Context, *, text=None): """ Change the name of the channels infochannel. @@ -405,7 +407,7 @@ class InfoChannel(Cog): if not await ctx.tick(): await ctx.send("Done!") - @channelname.command(name='online') + @channelname.command(name="online") async def _channelname_Online(self, ctx: commands.Context, *, text=None): """ Change the name of the online infochannel. @@ -429,7 +431,7 @@ class InfoChannel(Cog): if not await ctx.tick(): await ctx.send("Done!") - @channelname.command(name='offline') + @channelname.command(name="offline") async def _channelname_Offline(self, ctx: commands.Context, *, text=None): """ Change the name of the offline infochannel. @@ -453,11 +455,11 @@ class InfoChannel(Cog): if not await ctx.tick(): await ctx.send("Done!") - @channelname.command(name='role') + @channelname.command(name="role") async def _channelname_Role(self, ctx: commands.Context, *, text=None): """ Change the name of the infochannel for specific roles. - + All role infochannels follow this format. Do NOT confuse with the roles command that counts number of roles in the server @@ -501,17 +503,17 @@ class InfoChannel(Cog): if category is None: await self.config.guild(guild).category_id.set(None) category_id = None - + if category_id is None: category: discord.CategoryChannel = await guild.create_category( "Server Stats", reason="InfoChannel Category make" ) await self.config.guild(guild).category_id.set(category.id) - await category.edit(position = 0) + await category.edit(position=0) category_id = category.id - + category: discord.CategoryChannel = guild.get_channel(category_id) - + # Remove the old members channel first channel_id = await self.config.guild(guild).channel_id() if category_id is not None: @@ -538,7 +540,6 @@ class InfoChannel(Cog): ) await self.config.guild(guild).humanchannel_id.set(humanchannel.id) - # Remove the old bot channel first botchannel_id = await self.config.guild(guild).botchannel_id() if category_id is not None: @@ -552,7 +553,6 @@ class InfoChannel(Cog): ) await self.config.guild(guild).botchannel_id.set(botchannel.id) - # Remove the old roles channel first roleschannel_id = await self.config.guild(guild).roleschannel_id() if category_id is not None: @@ -567,7 +567,6 @@ class InfoChannel(Cog): ) await self.config.guild(guild).roleschannel_id.set(roleschannel.id) - # Remove the old channels channel first channels_channel_id = await self.config.guild(guild).channels_channel_id() if category_id is not None: @@ -580,7 +579,7 @@ class InfoChannel(Cog): "Total Channels:", reason="InfoChannel botcount", overwrites=overwrites ) await self.config.guild(guild).channels_channel_id.set(channels_channel.id) - + # Remove the old online channel first onlinechannel_id = await self.config.guild(guild).onlinechannel_id() if channel_id is not None: @@ -593,7 +592,7 @@ class InfoChannel(Cog): "Online:", reason="InfoChannel onlinecount", overwrites=overwrites ) await self.config.guild(guild).onlinechannel_id.set(onlinechannel.id) - + # Remove the old offline channel first offlinechannel_id = await self.config.guild(guild).offlinechannel_id() if channel_id is not None: @@ -608,7 +607,7 @@ class InfoChannel(Cog): await self.config.guild(guild).offlinechannel_id.set(offlinechannel.id) async with self.config.guild(guild).role_ids() as role_data: - #Remove the old role channels first + # Remove the old role channels first for role_id in role_data.keys(): role_channel_id = role_data[role_id] if role_channel_id is not None: @@ -616,17 +615,17 @@ class InfoChannel(Cog): if rolechannel: await rolechannel.delete(reason="InfoChannel delete") - #The actual toggle for a role counter + # The actual toggle for a role counter if role: if str(role.id) in role_data.keys(): - role_data.pop(str(role.id)) #if the role is there, then remove it + role_data.pop(str(role.id)) # if the role is there, then remove it else: - role_data[role.id] = None #No channel created yet but we want one to be made + role_data[role.id] = None # No channel created yet but we want one to be made if role_data: # Then create the new ones for role_id in role_data.keys(): rolechannel = await category.create_voice_channel( - str(role_id)+":", reason="InfoChannel rolecount", overwrites=overwrites + str(role_id) + ":", reason="InfoChannel rolecount", overwrites=overwrites ) role_data[role_id] = rolechannel.id @@ -672,7 +671,7 @@ class InfoChannel(Cog): rolechannel: discord.VoiceChannel = guild.get_channel(role_channel_id) if rolechannel: await rolechannel.delete(reason="InfoChannel delete") - + await self.config.guild(guild).clear() async def update_infochannel(self, guild: discord.Guild): @@ -693,12 +692,12 @@ class InfoChannel(Cog): bot_num = len([m for m in guild.members if m.bot]) # bot_msg = f"Bots: {num}" - #Gets count of roles in the server - roles_num = len(guild.roles)-1 + # Gets count of roles in the server + roles_num = len(guild.roles) - 1 # roles_msg = f"Total Roles: {num}" - #Gets count of channels in the server - # - - + # Gets count of channels in the server + # - - channels_num = len(guild.channels) - len(category.voice_channels) - len(guild.categories) # channels_msg = f"Total Channels: {num}" @@ -737,31 +736,31 @@ class InfoChannel(Cog): channel_names = await self.config.guild(guild).channel_names.all() if guild_data["member_count"]: - name = channel_names["members_channel"].format(count = members) + name = channel_names["members_channel"].format(count=members) await channel.edit(reason="InfoChannel update", name=name) if humancount: - name = channel_names["humans_channel"].format(count = human_num) + name = channel_names["humans_channel"].format(count=human_num) await humanchannel.edit(reason="InfoChannel update", name=name) if botcount: - name = channel_names["bots_channel"].format(count = bot_num) + name = channel_names["bots_channel"].format(count=bot_num) await botchannel.edit(reason="InfoChannel update", name=name) - + if rolescount: - name = channel_names["roles_channel"].format(count = roles_num) + name = channel_names["roles_channel"].format(count=roles_num) await roleschannel.edit(reason="InfoChannel update", name=name) - + if channelscount: - name = channel_names["channels_channel"].format(count = channels_num) + name = channel_names["channels_channel"].format(count=channels_num) await channels_channel.edit(reason="InfoChannel update", name=name) if onlinecount: - name = channel_names["online_channel"].format(count = online_num) + name = channel_names["online_channel"].format(count=online_num) await onlinechannel.edit(reason="InfoChannel update", name=name) - + if offlinecount: - name = channel_names["offline_channel"].format(count = offline) + name = channel_names["offline_channel"].format(count=offline) await offlinechannel.edit(reason="InfoChannel update", name=name) async with self.config.guild(guild).role_ids() as role_data: @@ -772,10 +771,9 @@ class InfoChannel(Cog): role_num = len(role.members) - name = channel_names["role_channel"].format(count = role_num, role = role.name) + name = channel_names["role_channel"].format(count=role_num, role=role.name) await rolechannel.edit(reason="InfoChannel update", name=name) - async def update_infochannel_with_cooldown(self, guild): """My attempt at preventing rate limits, lets see how it goes""" if self._critical_section_wooah_: @@ -842,26 +840,26 @@ class InfoChannel(Cog): channelscount = await self.config.guild(channel.guild).channels_count() if channelscount: await self.update_infochannel_with_cooldown(channel.guild) - - @Cog.listener() + + @Cog.listener() async def on_guild_role_create(self, role): if await self.bot.cog_disabled_in_guild(self, role.guild): return - + rolescount = await self.config.guild(role.guild).roles_count() if rolescount: await self.update_infochannel_with_cooldown(role.guild) - @Cog.listener() + @Cog.listener() async def on_guild_role_delete(self, role): if await self.bot.cog_disabled_in_guild(self, role.guild): return - + rolescount = await self.config.guild(role.guild).roles_count() if rolescount: await self.update_infochannel_with_cooldown(role.guild) - - #delete specific role counter if the role is deleted + + # delete specific role counter if the role is deleted async with self.config.guild(role.guild).role_ids() as role_data: if str(role.id) in role_data.keys(): role_channel_id = role_data[str(role.id)] From b566b58e1aad9a6637d8b7d12f6be81f1ec9937f Mon Sep 17 00:00:00 2001 From: bobloy Date: Tue, 8 Dec 2020 10:53:24 -0500 Subject: [PATCH 047/133] Infochannel rewrite --- infochannel/__init__.py | 8 +- infochannel/infochannel.py | 1171 +++++++++++++++--------------------- 2 files changed, 494 insertions(+), 685 deletions(-) diff --git a/infochannel/__init__.py b/infochannel/__init__.py index 514cd5f..1c4d081 100644 --- a/infochannel/__init__.py +++ b/infochannel/__init__.py @@ -1,5 +1,9 @@ +import asyncio + from .infochannel import InfoChannel -def setup(bot): - bot.add_cog(InfoChannel(bot)) +async def setup(bot): + ic_cog = InfoChannel(bot) + bot.add_cog(ic_cog) + await ic_cog.initialize() diff --git a/infochannel/infochannel.py b/infochannel/infochannel.py index f021c56..1b2bce4 100644 --- a/infochannel/infochannel.py +++ b/infochannel/infochannel.py @@ -1,25 +1,50 @@ import asyncio -from typing import Union +import logging +from collections import defaultdict +from typing import Dict, Optional, Union import discord from redbot.core import Config, checks, commands from redbot.core.bot import Red from redbot.core.commands import Cog -# Cog: Any = getattr(commands, "Cog", object) -# listener = getattr(commands.Cog, "listener", None) # Trusty + Sinbad -# if listener is None: -# def listener(name=None): -# return lambda x: x - -RATE_LIMIT_DELAY = 60 * 10 # If you're willing to risk rate limiting, you can decrease the delay +# 10 minutes. Rate limit is 2 per 10, so 1 per 6 is safe. +RATE_LIMIT_DELAY = 60 * 6 # If you're willing to risk rate limiting, you can decrease the delay + +log = logging.getLogger("red.fox_v3.infochannel") + + +async def get_channel_counts(category, guild): + # Gets count of bots + bot_num = len([m for m in guild.members if m.bot]) + # Gets count of roles in the server + roles_num = len(guild.roles) - 1 + # Gets count of channels in the server + # - - + channels_num = len(guild.channels) - len(category.voice_channels) - len(guild.categories) + # Gets all counts of members + members = guild.member_count + offline_num = len(list(filter(lambda m: m.status is discord.Status.offline, guild.members))) + online_num = members - offline_num + # Gets count of actual users + human_num = members - bot_num + return { + "members": members, + "humans": human_num, + "bots": bot_num, + "roles": roles_num, + "channels": channels_num, + "online": online_num, + "offline": offline_num, + } class InfoChannel(Cog): """ Create a channel with updating server info - Less important information about the cog + This relies on editing channels, which is a strictly rate-limited activity. + As such, updates will not be frequent. Currently capped at 1 per 5 minutes per server. """ def __init__(self, bot: Red): @@ -29,44 +54,55 @@ class InfoChannel(Cog): self, identifier=731101021116710497110110101108, force_registration=True ) + # self. so I can get the keys from this later + self.default_channel_names = { + "members": "Members: {count}", + "humans": "Humans: {count}", + "bots": "Bots: {count}", + "roles": "Roles: {count}", + "channels": "Channels: {count}", + "online": "Online: {count}", + "offline": "Offline: {count}", + } + + default_channel_ids = {k: None for k in self.default_channel_names.keys()} + # Only members is enabled by default + default_enabled_counts = {k: k == "members" for k in self.default_channel_names.keys()} + default_guild = { "category_id": None, - "channel_id": None, - "humanchannel_id": None, - "botchannel_id": None, - "roleschannel_id": None, - "channels_channel_id": None, - "onlinechannel_id": None, - "offlinechannel_id": None, - "role_ids": {}, - "member_count": True, - "human_count": False, - "bot_count": False, - "roles_count": False, - "channels_count": False, - "online_count": False, - "offline_count": False, - "channel_names": { - "category_name": "Server Stats", - "members_channel": "Total Members: {count}", - "humans_channel": "Humans: {count}", - "bots_channel": "Bots: {count}", - "roles_channel": "Total Roles: {count}", - "channels_channel": "Total Channels: {count}", - "online_channel": "Online: {count}", - "offline_channel": "Offline:{count}", - "role_channel": "{role}: {count}", - }, + "channel_ids": default_channel_ids, + "enabled_channels": default_enabled_counts, + "channel_names": self.default_channel_names, } self.config.register_guild(**default_guild) + self.default_role = {"enabled": False, "channel_id": None, "name": "{role}: {count}"} + + self.config.register_role(**self.default_role) + self._critical_section_wooah_ = 0 + self.channel_data = defaultdict(dict) + + self.edit_queue = defaultdict(lambda: defaultdict(lambda: asyncio.Queue(maxsize=2))) + + self._rate_limited_edits: Dict[int, Dict[str, Optional[asyncio.Task]]] = defaultdict( + lambda: defaultdict(lambda: None) + ) + async def red_delete_data_for_user(self, **kwargs): """Nothing to delete""" return + async def initialize(self): + for guild in self.bot.guilds: + await self.update_infochannel(guild) + + def cog_unload(self): + self.stop_all_queues() + @commands.command() @checks.admin() async def infochannel(self, ctx: commands.Context): @@ -89,29 +125,33 @@ class InfoChannel(Cog): category: Union[discord.CategoryChannel, None] = guild.get_channel(category_id) if category_id is not None and category is None: - await ctx.send("Info category has been deleted, recreate it?") + await ctx.maybe_send_embed("Info category has been deleted, recreate it?") elif category_id is None: - await ctx.send("Enable info channels on this server?") + await ctx.maybe_send_embed("Enable info channels on this server?") else: - await ctx.send("Do you wish to delete current info channels?") + await ctx.maybe_send_embed("Do you wish to delete current info channels?") msg = await self.bot.wait_for("message", check=check) if msg.content.upper() in ["N", "NO"]: - await ctx.send("Cancelled") + await ctx.maybe_send_embed("Cancelled") return if category is None: try: await self.make_infochannel(guild) except discord.Forbidden: - await ctx.send("Failure: Missing permission to create neccessary channels") + await ctx.maybe_send_embed( + "Failure: Missing permission to create necessary channels" + ) return else: await self.delete_all_infochannels(guild) + ctx.message = msg + if not await ctx.tick(): - await ctx.send("Done!") + await ctx.maybe_send_embed("Done!") @commands.group(aliases=["icset"]) @checks.admin() @@ -122,388 +162,189 @@ class InfoChannel(Cog): if not ctx.invoked_subcommand: pass - @infochannelset.command(name="membercount") - async def _infochannelset_membercount(self, ctx: commands.Context, enabled: bool = None): - """ - Toggle an infochannel that shows the amount of total members in the server - """ - guild = ctx.guild - if enabled is None: - enabled = not await self.config.guild(guild).member_count() - - await self.config.guild(guild).member_count.set(enabled) - await self.make_infochannel(ctx.guild) - - if enabled: - await ctx.send("InfoChannel for member count has been enabled.") - else: - await ctx.send("InfoChannel for member count has been disabled.") - - @infochannelset.command(name="humancount") - async def _infochannelset_humancount(self, ctx: commands.Context, enabled: bool = None): - """ - Toggle an infochannel that shows the amount of human users in the server - """ - guild = ctx.guild - if enabled is None: - enabled = not await self.config.guild(guild).human_count() - - await self.config.guild(guild).human_count.set(enabled) - await self.make_infochannel(ctx.guild) - - if enabled: - await ctx.send("InfoChannel for human user count has been enabled.") - else: - await ctx.send("InfoChannel for human user count has been disabled.") - - @infochannelset.command(name="botcount") - async def _infochannelset_botcount(self, ctx: commands.Context, enabled: bool = None): - """ - Toggle an infochannel that shows the amount of bots in the server - """ - guild = ctx.guild - if enabled is None: - enabled = not await self.config.guild(guild).bot_count() - - await self.config.guild(guild).bot_count.set(enabled) - await self.make_infochannel(ctx.guild) - - if enabled: - await ctx.send("InfoChannel for bot count has been enabled.") - else: - await ctx.send("InfoChannel for bot count has been disabled.") - - @infochannelset.command(name="rolescount") - async def _infochannelset_rolescount(self, ctx: commands.Context, enabled: bool = None): - """ - Toggle an infochannel that shows the amount of roles in the server - """ - guild = ctx.guild - if enabled is None: - enabled = not await self.config.guild(guild).roles_count() - - await self.config.guild(guild).roles_count.set(enabled) - await self.make_infochannel(ctx.guild) - - if enabled: - await ctx.send("InfoChannel for roles count has been enabled.") - else: - await ctx.send("InfoChannel for roles count has been disabled.") - - @infochannelset.command(name="channelscount") - async def _infochannelset_channelscount(self, ctx: commands.Context, enabled: bool = None): - """ - Toggle an infochannel that shows the amount of channels in the server - """ - guild = ctx.guild - if enabled is None: - enabled = not await self.config.guild(guild).channels_count() - - await self.config.guild(guild).channels_count.set(enabled) - await self.make_infochannel(ctx.guild) - - if enabled: - await ctx.send("InfoChannel for channels count has been enabled.") - else: - await ctx.send("InfoChannel for channels count has been disabled.") + @infochannelset.command(name="togglechannel") + async def _infochannelset_togglechannel( + self, ctx: commands.Context, channel_type: str, enabled: Optional[bool] = None + ): + """Toggles the infochannel for the specified channel type. - @infochannelset.command(name="onlinecount") - async def _infochannelset_onlinecount(self, ctx: commands.Context, enabled: bool = None): - """ - Toggle an infochannel that shows the amount of online users in the server + Valid Types are: + - `members`: Total members on the server + - `humans`: Total members that aren't bots + - `bots`: Total bots + - `roles`: Total number of roles + - `channels`: Total number of channels excluding infochannels, + - `online`: Total online members, + - `offline`: Total offline members, """ guild = ctx.guild - if enabled is None: - enabled = not await self.config.guild(guild).online_count() - - await self.config.guild(guild).online_count.set(enabled) - await self.make_infochannel(ctx.guild) - - if enabled: - await ctx.send("InfoChannel for online user count has been enabled.") - else: - await ctx.send("InfoChannel for online user count has been disabled.") + if channel_type not in self.default_channel_names.keys(): + await ctx.maybe_send_embed("Invalid channel type provided.") + return - @infochannelset.command(name="offlinecount") - async def _infochannelset_offlinecount(self, ctx: commands.Context, enabled: bool = None): - """ - Toggle an infochannel that shows the amount of offline users in the server - """ - guild = ctx.guild if enabled is None: - enabled = not await self.config.guild(guild).offline_count() + enabled = not await self.config.guild(guild).enabled_channels.get_raw(channel_type) - await self.config.guild(guild).offline_count.set(enabled) - await self.make_infochannel(ctx.guild) + await self.config.guild(guild).enabled_channels.set_raw(channel_type, value=enabled) + await self.make_infochannel(ctx.guild, channel_type=channel_type) if enabled: - await ctx.send("InfoChannel for offline user count has been enabled.") + await ctx.maybe_send_embed(f"InfoChannel `{channel_type}` has been enabled.") else: - await ctx.send("InfoChannel for offline user count has been disabled.") + await ctx.maybe_send_embed(f"InfoChannel `{channel_type}` has been disabled.") - @infochannelset.command(name="rolecount") + @infochannelset.command(name="togglerole") async def _infochannelset_rolecount( self, ctx: commands.Context, role: discord.Role, enabled: bool = None ): - """ - Toggle an infochannel that shows the amount of users in the server with the specified role - """ - guild = ctx.guild - role_data = await self.config.guild(guild).role_ids.all() + """Toggle an infochannel that shows the count of users with the specified role""" + if enabled is None: + enabled = not await self.config.role(role).enabled() - if str(role.id) in role_data.keys(): - enabled = False - else: - enabled = True + await self.config.role(role).enabled.set(enabled) - await self.make_infochannel(ctx.guild, role) + await self.make_infochannel(ctx.guild, channel_role=role) if enabled: - await ctx.send(f"InfoChannel for {role.name} count has been enabled.") + await ctx.maybe_send_embed(f"InfoChannel for {role.name} count has been enabled.") else: - await ctx.send(f"InfoChannel for {role.name} count has been disabled.") + await ctx.maybe_send_embed(f"InfoChannel for {role.name} count has been disabled.") - @infochannelset.group(name="name") - async def channelname(self, ctx: commands.Context): - """ - Change the name of the infochannels + @infochannelset.command(name="name") + async def _infochannelset_name(self, ctx: commands.Context, channel_type: str, *, text=None): """ - if not ctx.invoked_subcommand: - pass + Change the name of the infochannel for the specified channel type. - @channelname.command(name="category") - async def _channelname_Category(self, ctx: commands.Context, *, text): - """ - Change the name of the infochannel's category. - """ - guild = ctx.message.guild - category_id = await self.config.guild(guild).category_id() - category: discord.CategoryChannel = guild.get_channel(category_id) - await category.edit(name=text) - await self.config.guild(guild).channel_names.category_name.set(text) - if not await ctx.tick(): - await ctx.send("Done!") - - @channelname.command(name="members") - async def _channelname_Members(self, ctx: commands.Context, *, text=None): - """ - Change the name of the total members infochannel. - - {count} can be used to display number of total members in the server. - Leave blank to set back to default - Default is set to: - Total Members: {count} - - Example Formats: - Total Members: {count} - {count} Members - """ - guild = ctx.message.guild - if text: - await self.config.guild(guild).channel_names.members_channel.set(text) - else: - await self.config.guild(guild).channel_names.members_channel.clear() - - await self.update_infochannel(guild) - if not await ctx.tick(): - await ctx.send("Done!") - - @channelname.command(name="humans") - async def _channelname_Humans(self, ctx: commands.Context, *, text=None): - """ - Change the name of the human users infochannel. - - {count} can be used to display number of users in the server. - Leave blank to set back to default - Default is set to: - Humans: {count} + {count} must be used to display number of total members in the server. + Leave blank to set back to default. - Example Formats: - Users: {count} - {count} Users - """ - guild = ctx.message.guild - if text: - await self.config.guild(guild).channel_names.humans_channel.set(text) - else: - await self.config.guild(guild).channel_names.humans_channel.clear() - - await self.update_infochannel(guild) - if not await ctx.tick(): - await ctx.send("Done!") - - @channelname.command(name="bots") - async def _channelname_Bots(self, ctx: commands.Context, *, text=None): - """ - Change the name of the bots infochannel. + Examples: + - `[p]infochannelset name members Cool Cats: {count}` + - `[p]infochannelset name bots {count} Robot Overlords` - {count} can be used to display number of bots in the server. - Leave blank to set back to default - Default is set to: - Bots: {count} - - Example Formats: - Total Bots: {count} - {count} Robots - """ - guild = ctx.message.guild - if text: - await self.config.guild(guild).channel_names.bots_channel.set(text) - else: - await self.config.guild(guild).channel_names.bots_channel.clear() - - await self.update_infochannel(guild) - if not await ctx.tick(): - await ctx.send("Done!") + Valid Types are: + - `members`: Total members on the server + - `humans`: Total members that aren't bots + - `bots`: Total bots + - `roles`: Total number of roles + - `channels`: Total number of channels excluding infochannels + - `online`: Total online members + - `offline`: Total offline members - @channelname.command(name="roles") - async def _channelname_Roles(self, ctx: commands.Context, *, text=None): + Warning: This command counts against the channel update rate limit and may be queued. """ - Change the name of the roles infochannel. - - Do NOT confuse with the role command that counts number of members with a specified role - - {count} can be used to display number of roles in the server. - Leave blank to set back to default - Default is set to: - Total Roles: {count} + guild = ctx.guild + if channel_type not in self.default_channel_names.keys(): + await ctx.maybe_send_embed("Invalid channel type provided.") + return - Example Formats: - Total Roles: {count} - {count} Roles - """ - guild = ctx.message.guild - if text: - await self.config.guild(guild).channel_names.roles_channel.set(text) - else: - await self.config.guild(guild).channel_names.roles_channel.clear() + if text is None: + text = self.default_channel_names.get(channel_type) + elif "{count}" not in text: + await ctx.maybe_send_embed( + "Improperly formatted. Make sure to use `{count}` in your channel name" + ) + return + elif len(text) > 93: + await ctx.maybe_send_embed("Name is too long, max length is 93.") + return - await self.update_infochannel(guild) + await self.config.guild(guild).channel_names.set_raw(channel_type, value=text) + await self.update_infochannel(guild, channel_type=channel_type) if not await ctx.tick(): - await ctx.send("Done!") + await ctx.maybe_send_embed("Done!") - @channelname.command(name="channels") - async def _channelname_Channels(self, ctx: commands.Context, *, text=None): - """ - Change the name of the channels infochannel. - - {count} can be used to display number of channels in the server. - This does not count the infochannels - Leave blank to set back to default - Default is set to: - Total Channels: {count} - - Example Formats: - Total Channels: {count} - {count} Channels + @infochannelset.command(name="rolename") + async def _infochannelset_rolename( + self, ctx: commands.Context, role: discord.Role, *, text=None + ): """ - guild = ctx.message.guild - if text: - await self.config.guild(guild).channel_names.channels_channel.set(text) - else: - await self.config.guild(guild).channel_names.channels_channel.clear() + Change the name of the infochannel for specific roles. - await self.update_infochannel(guild) - if not await ctx.tick(): - await ctx.send("Done!") + {count} must be used to display number members with the given role. + {role} can be used for the roles name. + Leave blank to set back to default. - @channelname.command(name="online") - async def _channelname_Online(self, ctx: commands.Context, *, text=None): - """ - Change the name of the online infochannel. + Default is set to: `{role}: {count}` - {count} can be used to display number online members in the server. - Leave blank to set back to default - Default is set to: - Online: {count} + Examples: + - `[p]infochannelset rolename @Patrons {role}: {count}` + - `[p]infochannelset rolename Elite {count} members with {role} role` + - `[p]infochannelset rolename "Space Role" Total boosters: {count}` - Example Formats: - Total Online: {count} - {count} Online Members + Warning: This command counts against the channel update rate limit and may be queued. """ guild = ctx.message.guild - if text: - await self.config.guild(guild).channel_names.online_channel.set(text) - else: - await self.config.guild(guild).channel_names.online_channel.clear() - - await self.update_infochannel(guild) - if not await ctx.tick(): - await ctx.send("Done!") - - @channelname.command(name="offline") - async def _channelname_Offline(self, ctx: commands.Context, *, text=None): - """ - Change the name of the offline infochannel. - - {count} can be used to display number offline members in the server. - Leave blank to set back to default - Default is set to: - Offline: {count} - - Example Formats: - Total Offline: {count} - {count} Offline Members - """ - guild = ctx.message.guild - if text: - await self.config.guild(guild).channel_names.offline_channel.set(text) - else: - await self.config.guild(guild).channel_names.offline_channel.clear() + if text is None: + text = self.default_role["name"] + elif "{count}" not in text: + await ctx.maybe_send_embed( + "Improperly formatted. Make sure to use `{count}` in your channel name" + ) + return - await self.update_infochannel(guild) + await self.config.role(role).name.set(text) + await self.update_infochannel(guild, channel_role=role) if not await ctx.tick(): - await ctx.send("Done!") + await ctx.maybe_send_embed("Done!") - @channelname.command(name="role") - async def _channelname_Role(self, ctx: commands.Context, *, text=None): - """ - Change the name of the infochannel for specific roles. + async def create_individual_channel( + self, guild, category: discord.CategoryChannel, overwrites, channel_type, count + ): + # Delete the channel if it exists + channel_id = await self.config.guild(guild).channel_ids.get_raw(channel_type) + if channel_id is not None: + channel: discord.VoiceChannel = guild.get_channel(channel_id) + if channel: + self.stop_queue(guild.id, channel_type) + await channel.delete(reason="InfoChannel delete") - All role infochannels follow this format. - Do NOT confuse with the roles command that counts number of roles in the server + # Only make the channel if it's enabled + if await self.config.guild(guild).enabled_channels.get_raw(channel_type): + name = await self.config.guild(guild).channel_names.get_raw(channel_type) + name = name.format(count=count) + channel = await category.create_voice_channel( + name, reason="InfoChannel make", overwrites=overwrites + ) + await self.config.guild(guild).channel_ids.set_raw(channel_type, value=channel.id) + return channel + return None - {count} can be used to display number members with the given role. - {role} can be used for the roles name - Leave blank to set back to default - Default is set to: - {role}: {count} + async def create_role_channel( + self, guild, category: discord.CategoryChannel, overwrites, role: discord.Role + ): + # Delete the channel if it exists + channel_id = await self.config.role(role).channel_id() + if channel_id is not None: + channel: discord.VoiceChannel = guild.get_channel(channel_id) + if channel: + self.stop_queue(guild.id, role.id) + await channel.delete(reason="InfoChannel delete") - Example Formats: - {role}: {count} - {count} with {role} role - """ - guild = ctx.message.guild - if text: - await self.config.guild(guild).channel_names.role_channel.set(text) - else: - await self.config.guild(guild).channel_names.role_channel.clear() + # Only make the channel if it's enabled + if await self.config.role(role).enabled(): + count = len(role.members) + name = await self.config.role(role).name() + name = name.format(role=role.name, count=count) + channel = await category.create_voice_channel( + name, reason="InfoChannel make", overwrites=overwrites + ) + await self.config.role(role).channel_id.set(channel.id) + return channel + return None - await self.update_infochannel(guild) - if not await ctx.tick(): - await ctx.send("Done!") - - async def make_infochannel(self, guild: discord.Guild, role: discord.Role = None): - membercount = await self.config.guild(guild).member_count() - humancount = await self.config.guild(guild).human_count() - botcount = await self.config.guild(guild).bot_count() - rolescount = await self.config.guild(guild).roles_count() - channelscount = await self.config.guild(guild).channels_count() - onlinecount = await self.config.guild(guild).online_count() - offlinecount = await self.config.guild(guild).offline_count() + async def make_infochannel(self, guild: discord.Guild, channel_type=None, channel_role=None): overwrites = { guild.default_role: discord.PermissionOverwrite(connect=False), guild.me: discord.PermissionOverwrite(manage_channels=True, connect=True), } - # Check for and create the category + # Check for and create the Infochannel category category_id = await self.config.guild(guild).category_id() if category_id is not None: category: discord.CategoryChannel = guild.get_channel(category_id) - if category is None: - await self.config.guild(guild).category_id.set(None) + if category is None: # Category id is invalid, probably deleted. category_id = None - if category_id is None: category: discord.CategoryChannel = await guild.create_category( "Server Stats", reason="InfoChannel Category make" @@ -514,355 +355,319 @@ class InfoChannel(Cog): category: discord.CategoryChannel = guild.get_channel(category_id) - # Remove the old members channel first - channel_id = await self.config.guild(guild).channel_id() - if category_id is not None: - channel: discord.VoiceChannel = guild.get_channel(channel_id) - if channel: - await channel.delete(reason="InfoChannel delete") - if membercount: - # Then create the new one - channel = await category.create_voice_channel( - "Total Members:", reason="InfoChannel make", overwrites=overwrites - ) - await self.config.guild(guild).channel_id.set(channel.id) + channel_data = await get_channel_counts(category, guild) - # Remove the old human channel first - humanchannel_id = await self.config.guild(guild).humanchannel_id() - if category_id is not None: - humanchannel: discord.VoiceChannel = guild.get_channel(humanchannel_id) - if humanchannel: - await humanchannel.delete(reason="InfoChannel delete") - if humancount: - # Then create the new one - humanchannel = await category.create_voice_channel( - "Humans:", reason="InfoChannel humancount", overwrites=overwrites + # Only update a single channel + if channel_type is not None: + await self.create_individual_channel( + guild, category, overwrites, channel_type, channel_data[channel_type] ) - await self.config.guild(guild).humanchannel_id.set(humanchannel.id) + return + if channel_role is not None: + await self.create_role_channel(guild, category, overwrites, channel_role) + return - # Remove the old bot channel first - botchannel_id = await self.config.guild(guild).botchannel_id() - if category_id is not None: - botchannel: discord.VoiceChannel = guild.get_channel(botchannel_id) - if botchannel: - await botchannel.delete(reason="InfoChannel delete") - if botcount: - # Then create the new one - botchannel = await category.create_voice_channel( - "Bots:", reason="InfoChannel botcount", overwrites=overwrites + # Update all channels + for channel_type in self.default_channel_names.keys(): + await self.create_individual_channel( + guild, category, overwrites, channel_type, channel_data[channel_type] ) - await self.config.guild(guild).botchannel_id.set(botchannel.id) - # Remove the old roles channel first - roleschannel_id = await self.config.guild(guild).roleschannel_id() - if category_id is not None: - roleschannel: discord.VoiceChannel = guild.get_channel(roleschannel_id) - if roleschannel: - await roleschannel.delete(reason="InfoChannel delete") - - if rolescount: - # Then create the new one - roleschannel = await category.create_voice_channel( - "Total Roles:", reason="InfoChannel rolescount", overwrites=overwrites - ) - await self.config.guild(guild).roleschannel_id.set(roleschannel.id) + for role in guild.roles: + await self.create_role_channel(guild, category, overwrites, role) + + # await self.update_infochannel(guild) - # Remove the old channels channel first - channels_channel_id = await self.config.guild(guild).channels_channel_id() + async def delete_all_infochannels(self, guild: discord.Guild): + self.stop_guild_queues(guild.id) # Stop processing edits + + # Delete regular channels + for channel_type in self.default_channel_names.keys(): + channel_id = await self.config.guild(guild).channel_ids.get_raw(channel_type) + if channel_id is not None: + channel = guild.get_channel(channel_id) + if channel is not None: + await channel.delete(reason="InfoChannel delete") + await self.config.guild(guild).channel_ids.clear_raw(channel_type) + + # Delete role channels + for role in guild.roles: + channel_id = await self.config.role(role).channel_id() + if channel_id is not None: + channel = guild.get_channel(channel_id) + if channel is not None: + await channel.delete(reason="InfoChannel delete") + await self.config.role(role).channel_id.clear() + + # Delete the category last + category_id = await self.config.guild(guild).category_id() if category_id is not None: - channels_channel: discord.VoiceChannel = guild.get_channel(channels_channel_id) - if channels_channel: - await channels_channel.delete(reason="InfoChannel delete") - if channelscount: - # Then create the new one - channels_channel = await category.create_voice_channel( - "Total Channels:", reason="InfoChannel botcount", overwrites=overwrites - ) - await self.config.guild(guild).channels_channel_id.set(channels_channel.id) + category = guild.get_channel(category_id) + if category is not None: + await category.delete(reason="InfoChannel delete") - # Remove the old online channel first - onlinechannel_id = await self.config.guild(guild).onlinechannel_id() - if channel_id is not None: - onlinechannel: discord.VoiceChannel = guild.get_channel(onlinechannel_id) - if onlinechannel: - await onlinechannel.delete(reason="InfoChannel delete") - if onlinecount: - # Then create the new one - onlinechannel = await category.create_voice_channel( - "Online:", reason="InfoChannel onlinecount", overwrites=overwrites + async def add_to_queue(self, guild, channel, identifier, count, formatted_name): + self.channel_data[guild.id][identifier] = (count, formatted_name, channel.id) + if not self.edit_queue[guild.id][identifier].full(): + try: + self.edit_queue[guild.id][identifier].put_nowait(identifier) + except asyncio.QueueFull: + pass # If queue is full, disregard + + if self._rate_limited_edits[guild.id][identifier] is None: + await self.start_queue(guild.id, identifier) + + async def update_individual_channel(self, guild, channel_type, count, guild_data): + name = guild_data["channel_names"][channel_type] + name = name.format(count=count) + channel = guild.get_channel(guild_data["channel_ids"][channel_type]) + if channel is None: + return # abort + await self.add_to_queue(guild, channel, channel_type, count, name) + + async def update_role_channel(self, guild, role: discord.Role, role_data): + if not role_data["enabled"]: + return # Not enabled + count = len(role.members) + name = role_data["name"] + name = name.format(role=role.name, count=count) + channel = guild.get_channel(role_data["channel_id"]) + if channel is None: + return # abort + await self.add_to_queue(guild, channel, role.id, count, name) + + async def update_infochannel(self, guild: discord.Guild, channel_type=None, channel_role=None): + if channel_type is None and channel_role is None: + return await self.trigger_updates_for( + guild, + members=True, + humans=True, + bots=True, + roles=True, + channels=True, + online=True, + offline=True, + extra_roles=set(guild.roles), ) - await self.config.guild(guild).onlinechannel_id.set(onlinechannel.id) - # Remove the old offline channel first - offlinechannel_id = await self.config.guild(guild).offlinechannel_id() - if channel_id is not None: - offlinechannel: discord.VoiceChannel = guild.get_channel(offlinechannel_id) - if offlinechannel: - await offlinechannel.delete(reason="InfoChannel delete") - if offlinecount: - # Then create the new one - offlinechannel = await category.create_voice_channel( - "Offline:", reason="InfoChannel offlinecount", overwrites=overwrites - ) - await self.config.guild(guild).offlinechannel_id.set(offlinechannel.id) - - async with self.config.guild(guild).role_ids() as role_data: - # Remove the old role channels first - for role_id in role_data.keys(): - role_channel_id = role_data[role_id] - if role_channel_id is not None: - rolechannel: discord.VoiceChannel = guild.get_channel(role_channel_id) - if rolechannel: - await rolechannel.delete(reason="InfoChannel delete") - - # The actual toggle for a role counter - if role: - if str(role.id) in role_data.keys(): - role_data.pop(str(role.id)) # if the role is there, then remove it - else: - role_data[role.id] = None # No channel created yet but we want one to be made - if role_data: - # Then create the new ones - for role_id in role_data.keys(): - rolechannel = await category.create_voice_channel( - str(role_id) + ":", reason="InfoChannel rolecount", overwrites=overwrites - ) - role_data[role_id] = rolechannel.id + if channel_type is not None: + return await self.trigger_updates_for(guild, **{channel_type: True}) + + return await self.trigger_updates_for(guild, extra_roles=set(channel_role)) + + # category = guild.get_channel(await self.config.guild(guild).category_id()) + # + # if category is None: + # return # Nothing to update, must be off + # + # channel_counts = await get_channel_counts(category, guild) + # + # # Update individual channel + # if channel_type is not None: + # await self.update_individual_channel(guild, channel_type, channel_counts[channel_type]) + # return + # if channel_role is not None: + # await self.update_role_channel(guild, channel_role) + # return + # + # # Update all channels + # enabled_channels = await self.config.guild(guild).enabled_channels.all() + # for channel_type, enabled in enabled_channels.items(): + # if not enabled: + # continue + # await self.update_individual_channel(guild, channel_type, channel_counts[channel_type]) + # + # all_roles = await self.config.all_roles() + # guild_role_ids = set(role.id for role in guild.roles) + # for role_id, role_data in all_roles.values(): + # if int(role_id) not in guild_role_ids: + # continue + # role: discord.Role = guild.get_role(int(role_id)) + # await self.update_role_channel(guild, role) + + # def _start_timer(self, guild_id): + # self._stop_timer(guild_id) + # self._rate_limited_edits[guild_id] = asyncio.create_task( + # self.sleep_then_wakup(guild_id) + # ) + # + # async def sleep_then_wakup(self, guild_id, wait_seconds): + # await asyncio.sleep(wait_seconds) + # asyncio.create_task(self.wakeup(guild_id)) + # + # def _stop_timer(self, guild_id=None): + # if guild_id is None: + # for guild_id in self._timeout.keys(): + # if self._timeout[guild_id]: + # self._timeout[guild_id].cancel() + # self._timeout[guild_id] = None + # # del self._timeout + # else: + # if self._timeout[guild_id]: + # self._timeout[guild_id].cancel() + # self._timeout[guild_id] = None + # # del self._timeout[guild_id] + # + # async def wakeup(self, guild_id): + # self._stop_timer(guild_id) + # wait_seconds = await self._process_queue(guild_id) + # self._start_timer(guild_id, wait_seconds) + + async def start_queue(self, guild_id, identifier): + self._rate_limited_edits[guild_id][identifier] = asyncio.create_task( + self._process_queue(guild_id, identifier) + ) - await self.update_infochannel(guild) + def stop_queue(self, guild_id, identifier): + if self._rate_limited_edits[guild_id][identifier] is not None: + self._rate_limited_edits[guild_id][identifier].cancel() - async def delete_all_infochannels(self, guild: discord.Guild): - guild_data = await self.config.guild(guild).all() - role_data = guild_data["role_ids"] - category_id = guild_data["category_id"] - humanchannel_id = guild_data["humanchannel_id"] - botchannel_id = guild_data["botchannel_id"] - roleschannel_id = guild_data["roleschannel_id"] - channels_channel_id = guild_data["channels_channel_id"] - onlinechannel_id = guild_data["onlinechannel_id"] - offlinechannel_id = guild_data["offlinechannel_id"] - category: discord.CategoryChannel = guild.get_channel(category_id) - humanchannel: discord.VoiceChannel = guild.get_channel(humanchannel_id) - botchannel: discord.VoiceChannel = guild.get_channel(botchannel_id) - roleschannel: discord.VoiceChannel = guild.get_channel(roleschannel_id) - channels_channel: discord.VoiceChannel = guild.get_channel(channels_channel_id) - onlinechannel: discord.VoiceChannel = guild.get_channel(onlinechannel_id) - offlinechannel: discord.VoiceChannel = guild.get_channel(offlinechannel_id) - channel_id = guild_data["channel_id"] - channel: discord.VoiceChannel = guild.get_channel(channel_id) - await channel.delete(reason="InfoChannel delete") - if humanchannel_id is not None: - await humanchannel.delete(reason="InfoChannel delete") - if botchannel_id is not None: - await botchannel.delete(reason="InfoChannel delete") - if roleschannel_id is not None: - await roleschannel.delete(reason="InfoChannel delete") - if channels_channel is not None: - await channels_channel.delete(reason="InfoChannel delete") - if onlinechannel_id is not None: - await onlinechannel.delete(reason="InfoChannel delete") - if offlinechannel_id is not None: - await offlinechannel.delete(reason="InfoChannel delete") - if category_id is not None: - await category.delete(reason="InfoChannel delete") - async with self.config.guild(guild).role_ids() as role_data: - if role_data: - for role_channel_id in role_data.values(): - rolechannel: discord.VoiceChannel = guild.get_channel(role_channel_id) - if rolechannel: - await rolechannel.delete(reason="InfoChannel delete") + def stop_guild_queues(self, guild_id): + for identifier in self._rate_limited_edits[guild_id].keys(): + self.stop_queue(guild_id, identifier) + + def stop_all_queues(self): + for guild_id in self._rate_limited_edits.keys(): + self.stop_guild_queues(guild_id) - await self.config.guild(guild).clear() + async def _process_queue(self, guild_id, identifier): + while True: + identifier = await self.edit_queue[guild_id][identifier].get() # Waits forever - async def update_infochannel(self, guild: discord.Guild): + count, formatted_name, channel_id = self.channel_data[guild_id][identifier] + channel: discord.VoiceChannel = self.bot.get_channel(channel_id) + + if channel.name == formatted_name: + continue # Nothing to process + + log.debug(f"Processing guild_id: {guild_id} - identifier: {identifier}") + + try: + await channel.edit(reason="InfoChannel update", name=formatted_name) + except (discord.Forbidden, discord.HTTPException): + pass # Don't bother figuring it out + except discord.InvalidArgument: + log.exception(f"Invalid formatted infochannel: {formatted_name}") + else: + await asyncio.sleep(RATE_LIMIT_DELAY) # Wait a reasonable amount of time + + # async def update_infochannel_with_cooldown(self, guild): + # """My attempt at preventing rate limits, lets see how it goes""" + # if self._critical_section_wooah_: + # if self._critical_section_wooah_ == 2: + # # print("Already pending, skipping") + # return # Another one is already pending, don't queue more than one + # # print("Queuing another update") + # self._critical_section_wooah_ = 2 + # + # while self._critical_section_wooah_: + # await asyncio.sleep( + # RATE_LIMIT_DELAY // 4 + # ) # Max delay ends up as 1.25 * RATE_LIMIT_DELAY + # + # # print("Issuing queued update") + # return await self.update_infochannel_with_cooldown(guild) + # + # # print("Entering critical") + # self._critical_section_wooah_ = 1 + # await self.update_infochannel(guild) + # await asyncio.sleep(RATE_LIMIT_DELAY) + # self._critical_section_wooah_ = 0 + # # print("Exiting critical") + + # async def trigger_updates_for_all(self, guild_id): + # guild = self.bot.get_guild(guild_id) + # if guild is None: + # return None + # + # if await self.bot.cog_disabled_in_guild(self, guild): + # # Stop this background process if cog is disabled + # return None + # + # await self.update_infochannel(guild) # Refills the queue with updates + # return 30 + + async def trigger_updates_for(self, guild, **kwargs): + extra_roles: Optional[set] = kwargs.pop("extra_roles", False) guild_data = await self.config.guild(guild).all() - humancount = guild_data["human_count"] - botcount = guild_data["bot_count"] - rolescount = guild_data["roles_count"] - channelscount = guild_data["channels_count"] - onlinecount = guild_data["online_count"] - offlinecount = guild_data["offline_count"] - - category = guild.get_channel(guild_data["category_id"]) - - # Gets count of bots - # bots = lambda x: x.bot - # def bots(x): return x.bot - - bot_num = len([m for m in guild.members if m.bot]) - # bot_msg = f"Bots: {num}" - - # Gets count of roles in the server - roles_num = len(guild.roles) - 1 - # roles_msg = f"Total Roles: {num}" - - # Gets count of channels in the server - # - - - channels_num = len(guild.channels) - len(category.voice_channels) - len(guild.categories) - # channels_msg = f"Total Channels: {num}" - - # Gets all counts of members - members = guild.member_count - # member_msg = f"Total Members: {num}" - offline = len(list(filter(lambda m: m.status is discord.Status.offline, guild.members))) - # offline_msg = f"Offline: {num}" - online_num = members - offline - # online_msg = f"Online: {num}" - - # Gets count of actual users - total = lambda x: not x.bot - human_num = len([m for m in guild.members if total(m)]) - # human_msg = f"Users: {num}" - - channel_id = guild_data["channel_id"] - if channel_id is None: - return False - - botchannel_id = guild_data["botchannel_id"] - roleschannel_id = guild_data["roleschannel_id"] - channels_channel_id = guild_data["channels_channel_id"] - onlinechannel_id = guild_data["onlinechannel_id"] - offlinechannel_id = guild_data["offlinechannel_id"] - humanchannel_id = guild_data["humanchannel_id"] - channel_id = guild_data["channel_id"] - channel: discord.VoiceChannel = guild.get_channel(channel_id) - humanchannel: discord.VoiceChannel = guild.get_channel(humanchannel_id) - botchannel: discord.VoiceChannel = guild.get_channel(botchannel_id) - roleschannel: discord.VoiceChannel = guild.get_channel(roleschannel_id) - channels_channel: discord.VoiceChannel = guild.get_channel(channels_channel_id) - onlinechannel: discord.VoiceChannel = guild.get_channel(onlinechannel_id) - offlinechannel: discord.VoiceChannel = guild.get_channel(offlinechannel_id) - - channel_names = await self.config.guild(guild).channel_names.all() - - if guild_data["member_count"]: - name = channel_names["members_channel"].format(count=members) - await channel.edit(reason="InfoChannel update", name=name) - - if humancount: - name = channel_names["humans_channel"].format(count=human_num) - await humanchannel.edit(reason="InfoChannel update", name=name) - - if botcount: - name = channel_names["bots_channel"].format(count=bot_num) - await botchannel.edit(reason="InfoChannel update", name=name) - - if rolescount: - name = channel_names["roles_channel"].format(count=roles_num) - await roleschannel.edit(reason="InfoChannel update", name=name) - - if channelscount: - name = channel_names["channels_channel"].format(count=channels_num) - await channels_channel.edit(reason="InfoChannel update", name=name) - - if onlinecount: - name = channel_names["online_channel"].format(count=online_num) - await onlinechannel.edit(reason="InfoChannel update", name=name) - - if offlinecount: - name = channel_names["offline_channel"].format(count=offline) - await offlinechannel.edit(reason="InfoChannel update", name=name) - - async with self.config.guild(guild).role_ids() as role_data: - if role_data: - for role_id, role_channel_id in role_data.items(): - rolechannel: discord.VoiceChannel = guild.get_channel(role_channel_id) - role: discord.Role = guild.get_role(int(role_id)) - - role_num = len(role.members) - - name = channel_names["role_channel"].format(count=role_num, role=role.name) - await rolechannel.edit(reason="InfoChannel update", name=name) - - async def update_infochannel_with_cooldown(self, guild): - """My attempt at preventing rate limits, lets see how it goes""" - if self._critical_section_wooah_: - if self._critical_section_wooah_ == 2: - # print("Already pending, skipping") - return # Another one is already pending, don't queue more than one - # print("Queuing another update") - self._critical_section_wooah_ = 2 - - while self._critical_section_wooah_: - await asyncio.sleep( - RATE_LIMIT_DELAY // 4 - ) # Max delay ends up as 1.25 * RATE_LIMIT_DELAY - - # print("Issuing queued update") - return await self.update_infochannel_with_cooldown(guild) - - # print("Entering critical") - self._critical_section_wooah_ = 1 - await self.update_infochannel(guild) - await asyncio.sleep(RATE_LIMIT_DELAY) - self._critical_section_wooah_ = 0 - # print("Exiting critical") - @Cog.listener() - async def on_member_join(self, member: discord.Member): - if await self.bot.cog_disabled_in_guild(self, member.guild): - return - await self.update_infochannel_with_cooldown(member.guild) + to_update = ( + kwargs.keys() & guild_data["enabled_channels"].keys() + ) # Value in kwargs doesn't matter - @Cog.listener() - async def on_member_remove(self, member: discord.Member): + log.debug(f"{to_update=}") + + if to_update or extra_roles: + category = guild.get_channel(guild_data["category_id"]) + if category is None: + return # Nothing to update, must be off + + channel_data = await get_channel_counts(category, guild) + if to_update: + for channel_type in to_update: + await self.update_individual_channel( + guild, channel_type, channel_data[channel_type], guild_data + ) + if extra_roles: + role_data = await self.config.all_roles() + for channel_role in extra_roles: + if channel_role.id in role_data: + await self.update_role_channel( + guild, channel_role, role_data[channel_role.id] + ) + + @Cog.listener(name="on_member_join") + @Cog.listener(name="on_member_remove") + async def on_member_join_remove(self, member: discord.Member): if await self.bot.cog_disabled_in_guild(self, member.guild): return - await self.update_infochannel_with_cooldown(member.guild) + + if member.bot: + await self.trigger_updates_for( + member.guild, members=True, bots=True, online=True, offline=True + ) + else: + await self.trigger_updates_for( + member.guild, members=True, humans=True, online=True, offline=True + ) @Cog.listener() async def on_member_update(self, before: discord.Member, after: discord.Member): if await self.bot.cog_disabled_in_guild(self, after.guild): return - onlinecount = await self.config.guild(after.guild).online_count() - if onlinecount: - if before.status != after.status: - await self.update_infochannel_with_cooldown(after.guild) - role_data = await self.config.guild(after.guild).role_ids.all() - if role_data: - b = set(before.roles) - a = set(after.roles) - if b != a: - await self.update_infochannel_with_cooldown(after.guild) - @Cog.listener() - async def on_guild_channel_create(self, channel: discord.abc.GuildChannel): - if await self.bot.cog_disabled_in_guild(self, channel.guild): - return - channelscount = await self.config.guild(channel.guild).channels_count() - if channelscount: - await self.update_infochannel_with_cooldown(channel.guild) + if before.status != after.status: + return await self.trigger_updates_for(after.guild, online=True, offline=True) - @Cog.listener() - async def on_guild_channel_delete(self, channel: discord.abc.GuildChannel): + # XOR + c = set(after.roles) ^ set(before.roles) + + if c: + await self.trigger_updates_for(after.guild, extra_roles=c) + + @Cog.listener("on_guild_channel_create") + @Cog.listener("on_guild_channel_delete") + async def on_guild_channel_create_delete(self, channel: discord.TextChannel): if await self.bot.cog_disabled_in_guild(self, channel.guild): return - channelscount = await self.config.guild(channel.guild).channels_count() - if channelscount: - await self.update_infochannel_with_cooldown(channel.guild) + await self.trigger_updates_for(channel.guild, channels=True) @Cog.listener() async def on_guild_role_create(self, role): if await self.bot.cog_disabled_in_guild(self, role.guild): return - - rolescount = await self.config.guild(role.guild).roles_count() - if rolescount: - await self.update_infochannel_with_cooldown(role.guild) + await self.trigger_updates_for(role.guild, roles=True) @Cog.listener() async def on_guild_role_delete(self, role): if await self.bot.cog_disabled_in_guild(self, role.guild): return + await self.trigger_updates_for(role.guild, roles=True) - rolescount = await self.config.guild(role.guild).roles_count() - if rolescount: - await self.update_infochannel_with_cooldown(role.guild) - - # delete specific role counter if the role is deleted - async with self.config.guild(role.guild).role_ids() as role_data: - if str(role.id) in role_data.keys(): - role_channel_id = role_data[str(role.id)] - rolechannel: discord.VoiceChannel = role.guild.get_channel(role_channel_id) + role_channel_id = await self.config.role(role).channel_id() + if role_channel_id is not None: + rolechannel: discord.VoiceChannel = role.guild.get_channel(role_channel_id) + if rolechannel: await rolechannel.delete(reason="InfoChannel delete") - del role_data[str(role.id)] + + await self.config.role(role).clear() From c7820ec40c5ae6454c8b06de1a9ca6b3e75276f8 Mon Sep 17 00:00:00 2001 From: bobloy Date: Tue, 8 Dec 2020 10:54:59 -0500 Subject: [PATCH 048/133] No asyncio here --- infochannel/__init__.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/infochannel/__init__.py b/infochannel/__init__.py index 1c4d081..bbff901 100644 --- a/infochannel/__init__.py +++ b/infochannel/__init__.py @@ -1,5 +1,3 @@ -import asyncio - from .infochannel import InfoChannel From bf3c292fee9bc3275bc7382d6e002d211b8cfd2c Mon Sep 17 00:00:00 2001 From: bobloy Date: Tue, 8 Dec 2020 10:57:36 -0500 Subject: [PATCH 049/133] Black formatting --- fifo/redconfigjobstore.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fifo/redconfigjobstore.py b/fifo/redconfigjobstore.py index 126cfc0..51b3cdc 100644 --- a/fifo/redconfigjobstore.py +++ b/fifo/redconfigjobstore.py @@ -8,6 +8,7 @@ from apscheduler.jobstores.memory import MemoryJobStore from apscheduler.schedulers.asyncio import run_in_event_loop from apscheduler.util import datetime_to_utc_timestamp from redbot.core import Config + # TODO: use get_lock on config maybe from redbot.core.bot import Red from redbot.core.utils import AsyncIter @@ -26,7 +27,6 @@ class RedConfigJobStore(MemoryJobStore): self.pickle_protocol = pickle.HIGHEST_PROTOCOL self._eventloop = self.bot.loop # Used for @run_in_event_loop - @run_in_event_loop def start(self, scheduler, alias): super().start(scheduler, alias) From fc8e465c33cf14db3e51205b06d4db03eefbb18b Mon Sep 17 00:00:00 2001 From: bobloy Date: Tue, 8 Dec 2020 11:08:20 -0500 Subject: [PATCH 050/133] Remove excess comments --- infochannel/infochannel.py | 94 -------------------------------------- 1 file changed, 94 deletions(-) diff --git a/infochannel/infochannel.py b/infochannel/infochannel.py index 1b2bce4..9c9192c 100644 --- a/infochannel/infochannel.py +++ b/infochannel/infochannel.py @@ -455,64 +455,6 @@ class InfoChannel(Cog): return await self.trigger_updates_for(guild, extra_roles=set(channel_role)) - # category = guild.get_channel(await self.config.guild(guild).category_id()) - # - # if category is None: - # return # Nothing to update, must be off - # - # channel_counts = await get_channel_counts(category, guild) - # - # # Update individual channel - # if channel_type is not None: - # await self.update_individual_channel(guild, channel_type, channel_counts[channel_type]) - # return - # if channel_role is not None: - # await self.update_role_channel(guild, channel_role) - # return - # - # # Update all channels - # enabled_channels = await self.config.guild(guild).enabled_channels.all() - # for channel_type, enabled in enabled_channels.items(): - # if not enabled: - # continue - # await self.update_individual_channel(guild, channel_type, channel_counts[channel_type]) - # - # all_roles = await self.config.all_roles() - # guild_role_ids = set(role.id for role in guild.roles) - # for role_id, role_data in all_roles.values(): - # if int(role_id) not in guild_role_ids: - # continue - # role: discord.Role = guild.get_role(int(role_id)) - # await self.update_role_channel(guild, role) - - # def _start_timer(self, guild_id): - # self._stop_timer(guild_id) - # self._rate_limited_edits[guild_id] = asyncio.create_task( - # self.sleep_then_wakup(guild_id) - # ) - # - # async def sleep_then_wakup(self, guild_id, wait_seconds): - # await asyncio.sleep(wait_seconds) - # asyncio.create_task(self.wakeup(guild_id)) - # - # def _stop_timer(self, guild_id=None): - # if guild_id is None: - # for guild_id in self._timeout.keys(): - # if self._timeout[guild_id]: - # self._timeout[guild_id].cancel() - # self._timeout[guild_id] = None - # # del self._timeout - # else: - # if self._timeout[guild_id]: - # self._timeout[guild_id].cancel() - # self._timeout[guild_id] = None - # # del self._timeout[guild_id] - # - # async def wakeup(self, guild_id): - # self._stop_timer(guild_id) - # wait_seconds = await self._process_queue(guild_id) - # self._start_timer(guild_id, wait_seconds) - async def start_queue(self, guild_id, identifier): self._rate_limited_edits[guild_id][identifier] = asyncio.create_task( self._process_queue(guild_id, identifier) @@ -551,42 +493,6 @@ class InfoChannel(Cog): else: await asyncio.sleep(RATE_LIMIT_DELAY) # Wait a reasonable amount of time - # async def update_infochannel_with_cooldown(self, guild): - # """My attempt at preventing rate limits, lets see how it goes""" - # if self._critical_section_wooah_: - # if self._critical_section_wooah_ == 2: - # # print("Already pending, skipping") - # return # Another one is already pending, don't queue more than one - # # print("Queuing another update") - # self._critical_section_wooah_ = 2 - # - # while self._critical_section_wooah_: - # await asyncio.sleep( - # RATE_LIMIT_DELAY // 4 - # ) # Max delay ends up as 1.25 * RATE_LIMIT_DELAY - # - # # print("Issuing queued update") - # return await self.update_infochannel_with_cooldown(guild) - # - # # print("Entering critical") - # self._critical_section_wooah_ = 1 - # await self.update_infochannel(guild) - # await asyncio.sleep(RATE_LIMIT_DELAY) - # self._critical_section_wooah_ = 0 - # # print("Exiting critical") - - # async def trigger_updates_for_all(self, guild_id): - # guild = self.bot.get_guild(guild_id) - # if guild is None: - # return None - # - # if await self.bot.cog_disabled_in_guild(self, guild): - # # Stop this background process if cog is disabled - # return None - # - # await self.update_infochannel(guild) # Refills the queue with updates - # return 30 - async def trigger_updates_for(self, guild, **kwargs): extra_roles: Optional[set] = kwargs.pop("extra_roles", False) guild_data = await self.config.guild(guild).all() From 36826a44e7122216468b0e3941dc123329edede6 Mon Sep 17 00:00:00 2001 From: bobloy Date: Tue, 8 Dec 2020 11:17:50 -0500 Subject: [PATCH 051/133] Mostly line separators --- werewolf/werewolf.py | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/werewolf/werewolf.py b/werewolf/werewolf.py index f648569..d6be0b3 100644 --- a/werewolf/werewolf.py +++ b/werewolf/werewolf.py @@ -15,19 +15,11 @@ from werewolf.builder import ( role_from_id, role_from_name, ) -from werewolf.game import Game +from werewolf.game import Game, anyone_has_role log = logging.getLogger("red.fox_v3.werewolf") -async def anyone_has_role( - member_list: List[discord.Member], role: discord.Role -) -> Union[None, discord.Member]: - return await AsyncIter(member_list).find( - lambda m: AsyncIter(m.roles).find(lambda r: r.id == role.id) - ) - - class Werewolf(Cog): """ Base to host werewolf on a guild @@ -263,6 +255,7 @@ class Werewolf(Cog): game = await self._get_game(ctx) if not game: await ctx.maybe_send_embed("No game running, cannot start") + return if not await game.setup(ctx): pass # ToDo something? From 5892bed5b937054c4805977c0fb302545a0bb0ca Mon Sep 17 00:00:00 2001 From: bobloy Date: Tue, 8 Dec 2020 11:20:21 -0500 Subject: [PATCH 052/133] Didn't do werewolf label right --- .github/labeler.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/labeler.yml b/.github/labeler.yml index dd944c8..ecc1116 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -59,4 +59,4 @@ 'cog: unicode': - unicode/* 'cog: werewolf': - - werewolf \ No newline at end of file + - werewolf/* \ No newline at end of file From f3dab0f0c67a31b6118841ac0a019e81898c9ff4 Mon Sep 17 00:00:00 2001 From: bobloy Date: Fri, 18 Dec 2020 17:57:48 -0500 Subject: [PATCH 053/133] Fix construction of set --- infochannel/infochannel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infochannel/infochannel.py b/infochannel/infochannel.py index 9c9192c..33e2b10 100644 --- a/infochannel/infochannel.py +++ b/infochannel/infochannel.py @@ -453,7 +453,7 @@ class InfoChannel(Cog): if channel_type is not None: return await self.trigger_updates_for(guild, **{channel_type: True}) - return await self.trigger_updates_for(guild, extra_roles=set(channel_role)) + return await self.trigger_updates_for(guild, extra_roles={channel_role}) async def start_queue(self, guild_id, identifier): self._rate_limited_edits[guild_id][identifier] = asyncio.create_task( From b2c8268c9be00e31b9b8366dff17bb1d23339fdc Mon Sep 17 00:00:00 2001 From: bobloy Date: Tue, 22 Dec 2020 13:55:16 -0500 Subject: [PATCH 054/133] Update labeler --- .github/workflows/labeler.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml index 65e6640..82a4441 100644 --- a/.github/workflows/labeler.yml +++ b/.github/workflows/labeler.yml @@ -6,7 +6,7 @@ # https://github.com/actions/labeler name: Labeler -on: [pull_request] +on: [pull_request_target] jobs: label: From d13fd39cfcaad02d9f30b080bd26c1895fa427db Mon Sep 17 00:00:00 2001 From: bobloy Date: Tue, 22 Dec 2020 13:56:28 -0500 Subject: [PATCH 055/133] Require python-dateutil --- fifo/info.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fifo/info.json b/fifo/info.json index eb2a576..a690a92 100644 --- a/fifo/info.json +++ b/fifo/info.json @@ -10,7 +10,8 @@ "end_user_data_statement": "This cog does not store any End User Data", "requirements": [ "apscheduler", - "pytz" + "pytz", + "python-dateutil" ], "tags": [ "bobloy", From ce41c80c3b3f4235b669b461143c82b2e08ebe08 Mon Sep 17 00:00:00 2001 From: bobloy Date: Mon, 4 Jan 2021 08:50:48 -0500 Subject: [PATCH 056/133] Remove `fetch_message`, channel history is just better --- fifo/task.py | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/fifo/task.py b/fifo/task.py index 281b7d4..64f8ded 100644 --- a/fifo/task.py +++ b/fifo/task.py @@ -1,6 +1,6 @@ import logging from datetime import datetime, timedelta -from typing import Dict, List, Union +from typing import Dict, List, Optional, Union import discord from apscheduler.triggers.base import BaseTrigger @@ -269,20 +269,23 @@ class Task: ) return False - actual_message: discord.Message = channel.last_message + actual_message: Optional[discord.Message] = channel.last_message # I'd like to present you my chain of increasingly desperate message fetching attempts if actual_message is None: - # log.warning("No message found in channel cache yet, skipping execution") - # return - actual_message = await channel.fetch_message(channel.last_message_id) - if actual_message is None: # last_message_id was an invalid message I guess - actual_message = await channel.history(limit=1).flatten() - if not actual_message: # Basically only happens if the channel has no messages - actual_message = await author.history(limit=1).flatten() - if not actual_message: # Okay, the *author* has never sent a message? - log.warning("No message found in channel cache yet, skipping execution") - return False - actual_message = actual_message[0] + # Skip this one, never goes well + # try: + # actual_message = await channel.fetch_message(channel.last_message_id) + # except discord.NotFound: + # actual_message = None + # if actual_message is None: # last_message_id was an invalid message I guess + + actual_message = await channel.history(limit=1).flatten() + if not actual_message: # Basically only happens if the channel has no messages + actual_message = await author.history(limit=1).flatten() + if not actual_message: # Okay, the *author* has never sent a message? + log.warning("No message found in channel cache yet, skipping execution") + return False + actual_message = actual_message[0] message = FakeMessage(actual_message) # message = FakeMessage2 From d5bc5993ea2a0beacb9fab0039b1ceeb43460700 Mon Sep 17 00:00:00 2001 From: bobloy Date: Mon, 4 Jan 2021 09:14:44 -0500 Subject: [PATCH 057/133] Nevermind, bad idea. Just add the checks --- fifo/task.py | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/fifo/task.py b/fifo/task.py index 64f8ded..c02b68c 100644 --- a/fifo/task.py +++ b/fifo/task.py @@ -272,20 +272,21 @@ class Task: actual_message: Optional[discord.Message] = channel.last_message # I'd like to present you my chain of increasingly desperate message fetching attempts if actual_message is None: - # Skip this one, never goes well - # try: - # actual_message = await channel.fetch_message(channel.last_message_id) - # except discord.NotFound: - # actual_message = None - # if actual_message is None: # last_message_id was an invalid message I guess - - actual_message = await channel.history(limit=1).flatten() - if not actual_message: # Basically only happens if the channel has no messages - actual_message = await author.history(limit=1).flatten() - if not actual_message: # Okay, the *author* has never sent a message? - log.warning("No message found in channel cache yet, skipping execution") - return False - actual_message = actual_message[0] + # log.warning("No message found in channel cache yet, skipping execution") + # return + if channel.last_message_id is not None: + try: + actual_message = await channel.fetch_message(channel.last_message_id) + except discord.NotFound: + actual_message = None + if actual_message is None: # last_message_id was an invalid message I guess + actual_message = await channel.history(limit=1).flatten() + if not actual_message: # Basically only happens if the channel has no messages + actual_message = await author.history(limit=1).flatten() + if not actual_message: # Okay, the *author* has never sent a message? + log.warning("No message found in channel cache yet, skipping execution") + return False + actual_message = actual_message[0] message = FakeMessage(actual_message) # message = FakeMessage2 From 9c9b46dc7682f30a7260d79d65708b2ff3a43020 Mon Sep 17 00:00:00 2001 From: bobloy Date: Mon, 4 Jan 2021 14:47:59 -0500 Subject: [PATCH 058/133] Print expired triggers separately --- fifo/task.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/fifo/task.py b/fifo/task.py index c02b68c..53233c4 100644 --- a/fifo/task.py +++ b/fifo/task.py @@ -1,6 +1,6 @@ import logging from datetime import datetime, timedelta -from typing import Dict, List, Optional, Union +from typing import Dict, List, Optional, Tuple, Union import discord from apscheduler.triggers.base import BaseTrigger @@ -190,14 +190,23 @@ class Task: await self._decode_time_triggers() return self.data - async def get_triggers(self) -> List[BaseTrigger]: + async def get_triggers(self) -> Tuple[List[BaseTrigger], List[BaseTrigger]]: if not self.data: await self.load_from_config() if self.data is None or "triggers" not in self.data: # No triggers - return [] + return [], [] + + trigs = [] + expired_trigs = [] + for t in self.data["triggers"]: + trig = get_trigger(t) + if check_expired_trigger(trig): + expired_trigs.append(t) + else: + trigs.append(t) - return [get_trigger(t) for t in self.data["triggers"]] + return trigs, expired_trigs async def get_combined_trigger(self) -> Union[BaseTrigger, None]: if not self.data: From 320f729cc9c8ae7998284dc5763d153ce508cdbe Mon Sep 17 00:00:00 2001 From: bobloy Date: Mon, 4 Jan 2021 14:48:12 -0500 Subject: [PATCH 059/133] Print expired triggers separately --- fifo/fifo.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/fifo/fifo.py b/fifo/fifo.py index 8ff9c80..837caba 100644 --- a/fifo/fifo.py +++ b/fifo/fifo.py @@ -52,7 +52,7 @@ def _get_run_times(job: Job, now: datetime = None): if now is None: now = datetime(MAXYEAR, 12, 31, 23, 59, 59, 999999, tzinfo=job.next_run_time.tzinfo) - yield from _get_run_times(job, now) + yield from _get_run_times(job, now) # Recursion raise StopIteration() next_run_time = job.next_run_time @@ -235,6 +235,7 @@ class FIFO(commands.Cog): """Debug command to fix missed executions. If you see a negative "Next run time" when adding a trigger, this may help resolve it. + Check the logs when using this command. """ self.scheduler.wakeup() @@ -391,10 +392,14 @@ class FIFO(commands.Cog): else: embed.add_field(name="Server", value="Server not found", inline=False) + triggers, expired_triggers = await task.get_triggers() - trigger_str = "\n".join(str(t) for t in await task.get_triggers()) + trigger_str = "\n".join(str(t) for t in triggers) + expired_str = "\n".join(str(t) for t in expired_triggers) if trigger_str: embed.add_field(name="Triggers", value=trigger_str, inline=False) + if expired_str: + embed.add_field(name="Expired Triggers", value=expired_str, inline=False) job = await self._get_job(task) if job and job.next_run_time: From 9f10ea262d241f779b264cb225f6eaad7c5065c0 Mon Sep 17 00:00:00 2001 From: bobloy Date: Mon, 11 Jan 2021 08:52:57 -0500 Subject: [PATCH 060/133] Semantic change --- ccrole/ccrole.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ccrole/ccrole.py b/ccrole/ccrole.py index 59efc55..5248766 100644 --- a/ccrole/ccrole.py +++ b/ccrole/ccrole.py @@ -292,13 +292,13 @@ class CCRole(commands.Cog): # Thank you Cog-Creators cmd = ctx.invoked_with - cmd = cmd.lower() # Continues the proud case_insentivity tradition of ccrole + cmd = cmd.lower() # Continues the proud case-insensitivity tradition of ccrole guild = ctx.guild # message = ctx.message # Unneeded since switch to `on_message_without_command` from `on_command_error` - cmdlist = self.config.guild(guild).cmdlist + cmd_list = self.config.guild(guild).cmdlist # cmd = message.content[len(prefix) :].split()[0].lower() - cmd = await cmdlist.get_raw(cmd, default=None) + cmd = await cmd_list.get_raw(cmd, default=None) if cmd is not None: await self.eval_cc(cmd, message, ctx) From 2c9f3838da7bc8af2e70037ddf76017f535e59ea Mon Sep 17 00:00:00 2001 From: bobloy Date: Mon, 11 Jan 2021 09:05:41 -0500 Subject: [PATCH 061/133] Update info to include install instructions --- chatter/info.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chatter/info.json b/chatter/info.json index b79e587..85107ed 100644 --- a/chatter/info.json +++ b/chatter/info.json @@ -3,7 +3,7 @@ "Bobloy" ], "min_bot_version": "3.4.0", - "description": "Create an offline chatbot that talks like your average member using Machine Learning", + "description": "Create an offline chatbot that talks like your average member using Machine Learning. See setup instructions at https://github.com/bobloy/Fox-V3/tree/master/chatter", "hidden": false, "install_msg": "Thank you for installing Chatter! Please make sure you check the install instructions at https://github.com/bobloy/Fox-V3/blob/master/chatter/README.md\nAfter that, get started ith `[p]load chatter` and `[p]help Chatter`", "requirements": [ From d14db16746b6a37ded10dc4ad463ce544eb015ca Mon Sep 17 00:00:00 2001 From: bobloy Date: Mon, 11 Jan 2021 13:40:37 -0500 Subject: [PATCH 062/133] Small doc update --- werewolf/builder.py | 1 + 1 file changed, 1 insertion(+) diff --git a/werewolf/builder.py b/werewolf/builder.py index f57a669..da85b40 100644 --- a/werewolf/builder.py +++ b/werewolf/builder.py @@ -71,6 +71,7 @@ W1, W2, W5, W6 = Random Werewolf N1 = Benign Neutral 0001-1112T11W112N2 +which translates to 0,0,0,1,11,12,E1,R1,R1,R1,R2,P2 pre-letter = exact role position From 3b50785c5be698912e218a70d0438e66523ca147 Mon Sep 17 00:00:00 2001 From: bobloy Date: Mon, 11 Jan 2021 13:40:54 -0500 Subject: [PATCH 063/133] Docs update, better delete --- werewolf/werewolf.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/werewolf/werewolf.py b/werewolf/werewolf.py index d6be0b3..2074c48 100644 --- a/werewolf/werewolf.py +++ b/werewolf/werewolf.py @@ -50,15 +50,17 @@ class Werewolf(Cog): def cog_unload(self): log.debug("Unload called") - for game in self.games.values(): - del game + for key in self.games.keys(): + del self.games[key] @commands.command() async def buildgame(self, ctx: commands.Context): """ Create game codes to run custom games. - Pick the roles or randomized roles you want to include in a game + Pick the roles or randomized roles you want to include in a game. + + Note: The same role can be picked more than once. """ gb = GameBuilder() code = await gb.build_game(ctx) @@ -84,9 +86,6 @@ class Werewolf(Cog): Lists current guild settings """ valid, role, category, channel, log_channel = await self._get_settings(ctx) - # if not valid: - # await ctx.send("Failed to get settings") - # return None embed = discord.Embed( title="Current Guild Settings", From 9bdaf73944d094de4d2423566288681bf6c02010 Mon Sep 17 00:00:00 2001 From: bobloy Date: Mon, 11 Jan 2021 13:41:14 -0500 Subject: [PATCH 064/133] Add __str__ and TODO for better "seeing" --- werewolf/role.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/werewolf/role.py b/werewolf/role.py index e267283..90e5f5f 100644 --- a/werewolf/role.py +++ b/werewolf/role.py @@ -72,6 +72,9 @@ class Role(WolfListener): self.blocked = False self.properties = {} # Extra data for other roles (i.e. arsonist) + def __str__(self): + return self.__repr__() + def __repr__(self): return f"{self.__class__.__name__}({self.player.__repr__()})" @@ -86,7 +89,7 @@ class Role(WolfListener): log.debug(f"Assigned {self} to {player}") - async def get_alignment(self, source=None): + async def get_alignment(self, source=None): # TODO: Rework to be "strength" tiers """ Interaction for powerful access of alignment (Village, Werewolf, Other) From 6c669dd170d939663570f296f4032a73397c4bb7 Mon Sep 17 00:00:00 2001 From: bobloy Date: Thu, 14 Jan 2021 11:32:51 -0500 Subject: [PATCH 065/133] Change typing --- werewolf/werewolf.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/werewolf/werewolf.py b/werewolf/werewolf.py index 2074c48..a4083a9 100644 --- a/werewolf/werewolf.py +++ b/werewolf/werewolf.py @@ -1,11 +1,10 @@ import logging -from typing import List, Union +from typing import Optional import discord from redbot.core import Config, checks, commands from redbot.core.bot import Red from redbot.core.commands import Cog -from redbot.core.utils import AsyncIter from redbot.core.utils.menus import DEFAULT_CONTROLS, menu from werewolf.builder import ( @@ -392,7 +391,7 @@ class Werewolf(Cog): else: await ctx.maybe_send_embed("Role ID not found") - async def _get_game(self, ctx: commands.Context, game_code=None) -> Union[Game, None]: + async def _get_game(self, ctx: commands.Context, game_code=None) -> Optional[Game]: guild: discord.Guild = getattr(ctx, "guild", None) if guild is None: @@ -419,7 +418,7 @@ class Werewolf(Cog): return self.games[guild.id] - async def _game_start(self, game): + async def _game_start(self, game: Game): await game.start() async def _get_settings(self, ctx): From 796edb4d355d4498abccebdf2c8472d6283ed06a Mon Sep 17 00:00:00 2001 From: bobloy Date: Fri, 15 Jan 2021 13:46:37 -0500 Subject: [PATCH 066/133] Corrected expired triggers --- fifo/fifo.py | 24 +++++++++++++----------- fifo/task.py | 3 +-- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/fifo/fifo.py b/fifo/fifo.py index 837caba..89a9e83 100644 --- a/fifo/fifo.py +++ b/fifo/fifo.py @@ -146,14 +146,15 @@ class FIFO(commands.Cog): await task.delete_self() async def _process_task(self, task: Task): - job: Union[Job, None] = await self._get_job(task) - if job is not None: - combined_trigger_ = await task.get_combined_trigger() - if combined_trigger_ is None: - job.remove() - else: - job.reschedule(combined_trigger_) - return job + # None of this is necessar, we have `replace_existing` already + # job: Union[Job, None] = await self._get_job(task) + # if job is not None: + # combined_trigger_ = await task.get_combined_trigger() + # if combined_trigger_ is None: + # job.remove() + # else: + # job.reschedule(combined_trigger_) + # return job return await self._add_job(task) async def _get_job(self, task: Task) -> Job: @@ -173,9 +174,10 @@ class FIFO(commands.Cog): ) async def _resume_job(self, task: Task): - try: - job = self.scheduler.resume_job(job_id=_assemble_job_id(task.name, task.guild_id)) - except JobLookupError: + job: Union[Job, None] = await self._get_job(task) + if job is not None: + job.resume() + else: job = await self._process_task(task) return job diff --git a/fifo/task.py b/fifo/task.py index 53233c4..6f667b9 100644 --- a/fifo/task.py +++ b/fifo/task.py @@ -6,7 +6,6 @@ import discord from apscheduler.triggers.base import BaseTrigger from apscheduler.triggers.combining import OrTrigger from apscheduler.triggers.cron import CronTrigger -from apscheduler.triggers.date import DateTrigger from apscheduler.triggers.interval import IntervalTrigger from discord.utils import time_snowflake import pytz @@ -37,7 +36,7 @@ def get_trigger(data): def check_expired_trigger(trigger: BaseTrigger): - return trigger.get_next_fire_time(None, datetime.now(pytz.utc)) is not None + return trigger.get_next_fire_time(None, datetime.now(pytz.utc)) is None def parse_triggers(data: Union[Dict, None]): From 3f997fa804060c0efc8b8dd47827d32338d9e113 Mon Sep 17 00:00:00 2001 From: bobloy Date: Thu, 21 Jan 2021 10:44:25 -0500 Subject: [PATCH 067/133] Fix to pickle error and Nonetype comparison --- fifo/__init__.py | 10 ++++++++++ fifo/date_trigger.py | 5 ++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/fifo/__init__.py b/fifo/__init__.py index 34cfd7b..2d5e103 100644 --- a/fifo/__init__.py +++ b/fifo/__init__.py @@ -1,5 +1,15 @@ +import sys + from .fifo import FIFO +# Applying fix from: https://github.com/Azure/azure-functions-python-worker/issues/640 +# [Fix] Create a wrapper for importing imgres +from .date_trigger import * +from . import CustomDateTrigger + +# [Fix] Register imgres into system modules +sys.modules["CustomDateTrigger"] = CustomDateTrigger + async def setup(bot): cog = FIFO(bot) diff --git a/fifo/date_trigger.py b/fifo/date_trigger.py index eb3d617..b024750 100644 --- a/fifo/date_trigger.py +++ b/fifo/date_trigger.py @@ -4,4 +4,7 @@ from apscheduler.triggers.date import DateTrigger class CustomDateTrigger(DateTrigger): def get_next_fire_time(self, previous_fire_time, now): next_run = super().get_next_fire_time(previous_fire_time, now) - return next_run if next_run >= now else None + return next_run if next_run is not None and next_run >= now else None + + def __getstate__(self): + return {"version": 1, "run_date": self.run_date} From 6233db2272b75515ab1e7ff746745729bb369289 Mon Sep 17 00:00:00 2001 From: bobloy Date: Mon, 25 Jan 2021 17:04:20 -0500 Subject: [PATCH 068/133] FakeMessage is subclass and the implications --- fifo/fifo.py | 1 + fifo/task.py | 95 +++++++++++++++++++++++++++++++++++++++++++--------- 2 files changed, 81 insertions(+), 15 deletions(-) diff --git a/fifo/fifo.py b/fifo/fifo.py index 89a9e83..d152609 100644 --- a/fifo/fifo.py +++ b/fifo/fifo.py @@ -171,6 +171,7 @@ class FIFO(commands.Cog): id=_assemble_job_id(task.name, task.guild_id), trigger=combined_trigger_, name=task.name, + replace_existing=True, ) async def _resume_job(self, task: Task): diff --git a/fifo/task.py b/fifo/task.py index 6f667b9..1d325a4 100644 --- a/fifo/task.py +++ b/fifo/task.py @@ -3,12 +3,12 @@ from datetime import datetime, timedelta from typing import Dict, List, Optional, Tuple, Union import discord +import pytz from apscheduler.triggers.base import BaseTrigger from apscheduler.triggers.combining import OrTrigger from apscheduler.triggers.cron import CronTrigger from apscheduler.triggers.interval import IntervalTrigger from discord.utils import time_snowflake -import pytz from redbot.core import Config, commands from redbot.core.bot import Red @@ -56,10 +56,67 @@ def parse_triggers(data: Union[Dict, None]): return trigger -class FakeMessage: - def __init__(self, message: discord.Message): +# class FakeMessage: +# def __init__(self, message: discord.Message): +# d = {k: getattr(message, k, None) for k in dir(message)} +# self.__dict__.update(**d) + + +# Potential FakeMessage subclass of Message +# class DeleteSlots(type): +# @classmethod +# def __prepare__(metacls, name, bases): +# """Borrowed a bit from https://stackoverflow.com/q/56579348""" +# super_prepared = super().__prepare__(name, bases) +# print(super_prepared) +# return super_prepared + + +class FakeMessage(discord.Message): + def __init__(self, *args, message: discord.Message, **kwargs): d = {k: getattr(message, k, None) for k in dir(message)} - self.__dict__.update(**d) + for k, v in d.items(): + if k.lower().startswith("_handle"): + continue + try: + # log.debug(f"{k=} {v=}") + setattr(self, k, v) + except TypeError: + # log.exception("This is fine") + pass + except AttributeError: + # log.exception("This is fine") + pass + + self.id = time_snowflake(datetime.utcnow(), high=False) # Pretend to be now + self.type = discord.MessageType.default + + def process_the_rest( + self, + author: discord.Member, + channel: discord.TextChannel, + content, + ): + # self.content = content + # log.debug(self.content) + self._handle_content(content) + # log.debug(self.content) + + self.mention_everyone = "@everyone" in self.content or "@here" in self.content + + # for handler in ('author', 'member', 'mentions', 'mention_roles', 'call', 'flags'): + # try: + # getattr(self, '_handle_%s' % handler)(data[handler]) + # except KeyError: + # continue + self.author = author + # self._handle_author(author._user._to_minimal_user_json()) + # self._handle_member(author) + self._rebind_channel_reference(channel) + self._handle_mention_roles(self.raw_role_mentions) + self._handle_mentions(self.raw_mentions) + + # self.__dict__.update(**d) def neuter_message(message: FakeMessage): @@ -270,7 +327,7 @@ class Task: f"Could not execute Task[{self.name}] due to missing channel: {self.channel_id}" ) return False - author: discord.User = guild.get_member(self.author_id) + author: discord.Member = guild.get_member(self.author_id) if author is None: log.warning( f"Could not execute Task[{self.name}] due to missing author: {self.author_id}" @@ -296,22 +353,27 @@ class Task: return False actual_message = actual_message[0] - message = FakeMessage(actual_message) - # message = FakeMessage2 - message.author = author - message.guild = guild # Just in case we got desperate, see above - message.channel = channel - message.id = time_snowflake(datetime.utcnow(), high=False) # Pretend to be now - message = neuter_message(message) + # message._handle_author(author) # Option when message is subclass + # message._state = self.bot._get_state() + # Time to set the relevant attributes + # message.author = author + # Don't need guild with subclass, guild is just channel.guild + # message.guild = guild # Just in case we got desperate, see above + # message.channel = channel # absolutely weird that this takes a message object instead of guild - prefixes = await self.bot.get_prefix(message) + prefixes = await self.bot.get_prefix(actual_message) if isinstance(prefixes, str): prefix = prefixes else: prefix = prefixes[0] - message.content = f"{prefix}{self.get_command_str()}" + new_content = f"{prefix}{self.get_command_str()}" + # log.debug(f"{new_content=}") + + message = FakeMessage(message=actual_message) + message = neuter_message(message) + message.process_the_rest(author=author, channel=channel, content=new_content) if ( not message.guild @@ -319,7 +381,10 @@ class Task: or not message.content or message.content == prefix ): - log.warning(f"Could not execute Task[{self.name}] due to message problem: {message}") + log.warning( + f"Could not execute Task[{self.name}] due to message problem: " + f"{message.guild=}, {message.author=}, {message.content=}" + ) return False new_ctx: commands.Context = await self.bot.get_context(message) From dbf6ba5a4b9be6cb0f9ff9d23fda1e496acd60b2 Mon Sep 17 00:00:00 2001 From: bobloy Date: Tue, 26 Jan 2021 17:10:47 -0500 Subject: [PATCH 069/133] More precise message imitation --- fifo/task.py | 50 +++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 41 insertions(+), 9 deletions(-) diff --git a/fifo/task.py b/fifo/task.py index 1d325a4..cd7200d 100644 --- a/fifo/task.py +++ b/fifo/task.py @@ -71,13 +71,32 @@ def parse_triggers(data: Union[Dict, None]): # print(super_prepared) # return super_prepared +things_for_fakemessage_to_steal = [ + "_state", + "id", + "webhook_id", + "reactions", + "attachments", + "embeds", + "application", + "activity", + "channel", + "_edited_time", + "type", + "pinned", + "flags", + "mention_everyone", + "tts", + "content", + "nonce", + "reference", +] + class FakeMessage(discord.Message): def __init__(self, *args, message: discord.Message, **kwargs): - d = {k: getattr(message, k, None) for k in dir(message)} + d = {k: getattr(message, k, None) for k in things_for_fakemessage_to_steal} for k, v in d.items(): - if k.lower().startswith("_handle"): - continue try: # log.debug(f"{k=} {v=}") setattr(self, k, v) @@ -99,10 +118,6 @@ class FakeMessage(discord.Message): ): # self.content = content # log.debug(self.content) - self._handle_content(content) - # log.debug(self.content) - - self.mention_everyone = "@everyone" in self.content or "@here" in self.content # for handler in ('author', 'member', 'mentions', 'mention_roles', 'call', 'flags'): # try: @@ -113,8 +128,25 @@ class FakeMessage(discord.Message): # self._handle_author(author._user._to_minimal_user_json()) # self._handle_member(author) self._rebind_channel_reference(channel) - self._handle_mention_roles(self.raw_role_mentions) - self._handle_mentions(self.raw_mentions) + self._update( + { + "content": content, + } + ) + self._update( + { + "mention_roles": self.raw_role_mentions, + "mentions": self.raw_mentions, + } + ) + + # self._handle_content(content) + # log.debug(self.content) + + self.mention_everyone = "@everyone" in self.content or "@here" in self.content + + # self._handle_mention_roles(self.raw_role_mentions) + # self._handle_mentions(self.raw_mentions) # self.__dict__.update(**d) From 14f8b825d8a5a81d12aa885da7236b13e97964d0 Mon Sep 17 00:00:00 2001 From: bobloy Date: Tue, 2 Feb 2021 16:35:41 -0500 Subject: [PATCH 070/133] Fix bad learning and checks --- chatter/chat.py | 55 ++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 47 insertions(+), 8 deletions(-) diff --git a/chatter/chat.py b/chatter/chat.py index 0988d46..a0a5f28 100644 --- a/chatter/chat.py +++ b/chatter/chat.py @@ -2,8 +2,10 @@ import asyncio import logging import os import pathlib +from collections import defaultdict from datetime import datetime, timedelta -from typing import Optional +from functools import partial +from typing import Dict, Optional import discord from chatterbot import ChatBot @@ -75,6 +77,10 @@ class Chatter(Cog): self.loop = asyncio.get_event_loop() + self._guild_cache = defaultdict(dict) + + self._last_message_per_channel: Dict[Optional[discord.Message]] = defaultdict(lambda: None) + async def red_delete_data_for_user(self, **kwargs): """Nothing to delete""" return @@ -190,6 +196,7 @@ class Chatter(Cog): if ctx.invoked_subcommand is None: pass + @commands.admin() @chatter.command(name="channel") async def chatter_channel( self, ctx: commands.Context, channel: Optional[discord.TextChannel] = None @@ -209,6 +216,7 @@ class Chatter(Cog): await self.config.guild(ctx.guild).chatchannel.set(channel.id) await ctx.maybe_send_embed(f"Chat channel is now {channel.mention}") + @commands.is_owner() @chatter.command(name="cleardata") async def chatter_cleardata(self, ctx: commands.Context, confirm: bool = False): """ @@ -241,6 +249,7 @@ class Chatter(Cog): await ctx.tick() + @commands.is_owner() @chatter.command(name="algorithm", aliases=["algo"]) async def chatter_algorithm( self, ctx: commands.Context, algo_number: int, threshold: float = None @@ -274,6 +283,7 @@ class Chatter(Cog): await ctx.tick() + @commands.is_owner() @chatter.command(name="model") async def chatter_model(self, ctx: commands.Context, model_number: int): """ @@ -311,6 +321,7 @@ class Chatter(Cog): f"Model has been switched to {self.tagger_language.ISO_639_1}" ) + @commands.is_owner() @chatter.command(name="minutes") async def minutes(self, ctx: commands.Context, minutes: int): """ @@ -322,10 +333,12 @@ class Chatter(Cog): await ctx.send_help() return - await self.config.guild(ctx.guild).convo_length.set(minutes) + await self.config.guild(ctx.guild).convo_delta.set(minutes) + self._guild_cache[ctx.guild.id]["convo_delta"] = minutes await ctx.tick() + @commands.is_owner() @chatter.command(name="age") async def age(self, ctx: commands.Context, days: int): """ @@ -340,6 +353,7 @@ class Chatter(Cog): await self.config.guild(ctx.guild).days.set(days) await ctx.tick() + @commands.is_owner() @chatter.command(name="backup") async def backup(self, ctx, backupname): """ @@ -361,6 +375,7 @@ class Chatter(Cog): else: await ctx.maybe_send_embed("Error occurred :(") + @commands.is_owner() @chatter.command(name="trainubuntu") async def chatter_train_ubuntu(self, ctx: commands.Context, confirmation: bool = False): """ @@ -382,6 +397,7 @@ class Chatter(Cog): else: await ctx.send("Error occurred :(") + @commands.is_owner() @chatter.command(name="trainenglish") async def chatter_train_english(self, ctx: commands.Context): """ @@ -395,6 +411,7 @@ class Chatter(Cog): else: await ctx.maybe_send_embed("Error occurred :(") + @commands.is_owner() @chatter.command() async def train(self, ctx: commands.Context, channel: discord.TextChannel): """ @@ -477,12 +494,34 @@ class Chatter(Cog): text = message.clean_content - async with channel.typing(): - # Switched to `generate_response` from `get_result` - # Switch back once better conversation detection is used. - future = await self.loop.run_in_executor(None, self.chatbot.generate_response, text) + async with ctx.typing(): + + if not self._guild_cache[ctx.guild.id]: + self._guild_cache[ctx.guild.id] = await self.config.guild(ctx.guild).all() + + if self._last_message_per_channel[ctx.channel.id] is not None: + last_m: discord.Message = self._last_message_per_channel[ctx.channel.id] + minutes = self._guild_cache[ctx.guild.id]["convo_delta"] + if (datetime.utcnow() - last_m.created_at).seconds > minutes*60: + in_response_to = None + else: + in_response_to = last_m.content + else: + in_response_to = None + + if in_response_to is None: + log.debug("Generating response") + Statement = self.chatbot.storage.get_object('statement') + future = await self.loop.run_in_executor( + None, self.chatbot.generate_response, Statement(text) + ) + else: + log.debug("Getting response") + future = await self.loop.run_in_executor( + None, partial(self.chatbot.get_response, text, in_response_to=in_response_to) + ) if future and str(future): - await channel.send(str(future)) + self._last_message_per_channel[ctx.channel.id] = await ctx.send(str(future)) else: - await channel.send(":thinking:") + await ctx.send(":thinking:") From 337def2fa32ebdcc788e5d785bd388537a6b4899 Mon Sep 17 00:00:00 2001 From: bobloy Date: Mon, 15 Feb 2021 10:18:18 -0500 Subject: [PATCH 071/133] Some progress on updated ubuntu trainer --- chatter/chat.py | 79 +++++++++++++++++++++--- chatter/info.json | 3 +- chatter/trainers.py | 142 +++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 213 insertions(+), 11 deletions(-) diff --git a/chatter/chat.py b/chatter/chat.py index a0a5f28..098ba73 100644 --- a/chatter/chat.py +++ b/chatter/chat.py @@ -17,7 +17,7 @@ from redbot.core.commands import Cog from redbot.core.data_manager import cog_data_path from redbot.core.utils.predicates import MessagePredicate -from chatter.trainers import TwitterCorpusTrainer +from chatter.trainers import TwitterCorpusTrainer, UbuntuCorpusTrainer2 log = logging.getLogger("red.fox_v3.chatter") @@ -168,6 +168,10 @@ class Chatter(Cog): trainer.train() return True + async def _train_ubuntu2(self): + trainer = UbuntuCorpusTrainer2(self.chatbot, cog_data_path(self)) + await trainer.asynctrain() + def _train_english(self): trainer = ChatterBotCorpusTrainer(self.chatbot) # try: @@ -353,6 +357,15 @@ class Chatter(Cog): await self.config.guild(ctx.guild).days.set(days) await ctx.tick() + @commands.is_owner() + @chatter.command(name="kaggle") + async def chatter_kaggle(self, ctx: commands.Context): + """Register with the kaggle API to download additional datasets for training""" + if not await self.check_for_kaggle(): + await ctx.maybe_send_embed( + "[Click here for instructions to setup the kaggle api](https://github.com/Kaggle/kaggle-api#api-credentials)" + ) + @commands.is_owner() @chatter.command(name="backup") async def backup(self, ctx, backupname): @@ -376,7 +389,13 @@ class Chatter(Cog): await ctx.maybe_send_embed("Error occurred :(") @commands.is_owner() - @chatter.command(name="trainubuntu") + @chatter.group(name="train") + async def chatter_train(self, ctx: commands.Context): + """Commands for training the bot""" + pass + + @commands.is_owner() + @chatter_train.command(name="ubuntu") async def chatter_train_ubuntu(self, ctx: commands.Context, confirmation: bool = False): """ WARNING: Large Download! Trains the bot using Ubuntu Dialog Corpus data. @@ -385,7 +404,7 @@ class Chatter(Cog): if not confirmation: await ctx.maybe_send_embed( "Warning: This command downloads ~500MB then eats your CPU for training\n" - "If you're sure you want to continue, run `[p]chatter trainubuntu True`" + "If you're sure you want to continue, run `[p]chatter train ubuntu True`" ) return @@ -398,7 +417,29 @@ class Chatter(Cog): await ctx.send("Error occurred :(") @commands.is_owner() - @chatter.command(name="trainenglish") + @chatter_train.command(name="ubuntu2") + async def chatter_train_ubuntu2(self, ctx: commands.Context, confirmation: bool = False): + """ + WARNING: Large Download! Trains the bot using *NEW* Ubuntu Dialog Corpus data. + """ + + if not confirmation: + await ctx.maybe_send_embed( + "Warning: This command downloads ~800 then eats your CPU for training\n" + "If you're sure you want to continue, run `[p]chatter train ubuntu2 True`" + ) + return + + async with ctx.typing(): + future = await self._train_ubuntu2() + + if future: + await ctx.send("Training successful!") + else: + await ctx.send("Error occurred :(") + + @commands.is_owner() + @chatter_train.command(name="english") async def chatter_train_english(self, ctx: commands.Context): """ Trains the bot in english @@ -412,10 +453,27 @@ class Chatter(Cog): await ctx.maybe_send_embed("Error occurred :(") @commands.is_owner() - @chatter.command() - async def train(self, ctx: commands.Context, channel: discord.TextChannel): + @chatter_train.command(name="list") + async def chatter_train_list(self, ctx: commands.Context): + """Trains the bot based on an uploaded list. + + Must be a file in the format of a python list: ['prompt', 'response1', 'response2'] + """ + if not ctx.message.attachments: + await ctx.maybe_send_embed("You must upload a file when using this command") + return + + attachment: discord.Attachment = ctx.message.attachments[0] + + a_bytes = await attachment.read() + + await ctx.send("Not yet implemented") + + @commands.is_owner() + @chatter_train.command(name="channel") + async def chatter_train_channel(self, ctx: commands.Context, channel: discord.TextChannel): """ - Trains the bot based on language in this guild + Trains the bot based on language in this guild. """ await ctx.maybe_send_embed( @@ -502,7 +560,7 @@ class Chatter(Cog): if self._last_message_per_channel[ctx.channel.id] is not None: last_m: discord.Message = self._last_message_per_channel[ctx.channel.id] minutes = self._guild_cache[ctx.guild.id]["convo_delta"] - if (datetime.utcnow() - last_m.created_at).seconds > minutes*60: + if (datetime.utcnow() - last_m.created_at).seconds > minutes * 60: in_response_to = None else: in_response_to = last_m.content @@ -511,7 +569,7 @@ class Chatter(Cog): if in_response_to is None: log.debug("Generating response") - Statement = self.chatbot.storage.get_object('statement') + Statement = self.chatbot.storage.get_object("statement") future = await self.loop.run_in_executor( None, self.chatbot.generate_response, Statement(text) ) @@ -525,3 +583,6 @@ class Chatter(Cog): self._last_message_per_channel[ctx.channel.id] = await ctx.send(str(future)) else: await ctx.send(":thinking:") + + async def check_for_kaggle(self): + return False diff --git a/chatter/info.json b/chatter/info.json index b79e587..a048c23 100644 --- a/chatter/info.json +++ b/chatter/info.json @@ -17,7 +17,8 @@ "pytz", "https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-2.3.1/en_core_web_sm-2.3.1.tar.gz#egg=en_core_web_sm", "https://github.com/explosion/spacy-models/releases/download/en_core_web_md-2.3.1/en_core_web_md-2.3.1.tar.gz#egg=en_core_web_md", - "spacy>=2.3,<2.4" + "spacy>=2.3,<2.4", + "kaggle" ], "short": "Local Chatbot run on machine learning", "end_user_data_statement": "This cog only stores anonymous conversations data; no End User Data is stored.", diff --git a/chatter/trainers.py b/chatter/trainers.py index 42d6288..0b765b7 100644 --- a/chatter/trainers.py +++ b/chatter/trainers.py @@ -1,6 +1,146 @@ +import asyncio +import csv +import logging +import os +import pathlib +import time +from functools import partial + from chatterbot import utils from chatterbot.conversation import Statement +from chatterbot.tagging import PosLemmaTagger from chatterbot.trainers import Trainer +from redbot.core.bot import Red +from dateutil import parser as date_parser +from redbot.core.utils import AsyncIter + +log = logging.getLogger("red.fox_v3.chatter.trainers") + + +class KaggleTrainer(Trainer): + def __init__(self, chatbot, datapath: pathlib.Path, **kwargs): + super().__init__(chatbot, **kwargs) + + self.data_directory = datapath / kwargs.get("downloadpath", "kaggle_download") + + self.kaggle_dataset = kwargs.get( + "kaggle_dataset", + "Cornell-University/movie-dialog-corpus", + ) + + # Create the data directory if it does not already exist + if not os.path.exists(self.data_directory): + os.makedirs(self.data_directory) + + def is_downloaded(self, file_path): + """ + Check if the data file is already downloaded. + """ + if os.path.exists(file_path): + self.chatbot.logger.info("File is already downloaded") + return True + + return False + + async def download(self, dataset): + import kaggle # This triggers the API token check + + future = await asyncio.get_event_loop().run_in_executor( + None, + partial( + kaggle.api.dataset_download_files, + dataset=dataset, + path=self.data_directory, + quiet=False, + unzip=True, + ), + ) + + +class UbuntuCorpusTrainer2(KaggleTrainer): + def __init__(self, chatbot, datapath: pathlib.Path, **kwargs): + super().__init__( + chatbot, + datapath, + downloadpath="ubuntu_data_v2", + kaggle_dataset="rtatman/ubuntu-dialogue-corpus", + **kwargs + ) + + async def asynctrain(self, *args, **kwargs): + extracted_dir = self.data_directory / "Ubuntu-dialogue-corpus" + + # Download and extract the Ubuntu dialog corpus if needed + if not extracted_dir.exists(): + await self.download(self.kaggle_dataset) + else: + log.info("Ubuntu dialogue already downloaded") + if not extracted_dir.exists(): + raise FileNotFoundError("Did not extract in the expected way") + + train_dialogue = kwargs.get("train_dialogue", True) + train_196_dialogue = kwargs.get("train_196", False) + train_301_dialogue = kwargs.get("train_301", False) + + if train_dialogue: + await self.run_dialogue_training(extracted_dir, "dialogueText.csv") + + if train_196_dialogue: + await self.run_dialogue_training(extracted_dir, "dialogueText_196.csv") + + if train_301_dialogue: + await self.run_dialogue_training(extracted_dir, "dialogueText_301.csv") + + async def run_dialogue_training(self, extracted_dir, dialogue_file): + log.info(f"Beginning dialogue training on {dialogue_file}") + start_time = time.time() + + tagger = PosLemmaTagger(language=self.chatbot.storage.tagger.language) + + with open(extracted_dir / dialogue_file, "r", encoding="utf-8") as dg: + reader = csv.DictReader(dg) + + next(reader) # Skip the header + + last_dialogue_id = None + previous_statement_text = None + previous_statement_search_text = "" + statements_from_file = [] + + async for row in AsyncIter(reader): + dialogue_id = row["dialogueID"] + if dialogue_id != last_dialogue_id: + previous_statement_text = None + previous_statement_search_text = "" + last_dialogue_id = dialogue_id + + if len(row) > 0: + statement = Statement( + text=row["text"], + in_response_to=previous_statement_text, + conversation="training", + created_at=date_parser.parse(row["date"]), + persona=row["from"], + ) + + for preprocessor in self.chatbot.preprocessors: + statement = preprocessor(statement) + + statement.search_text = tagger.get_text_index_string(statement.text) + statement.search_in_response_to = previous_statement_search_text + + previous_statement_text = statement.text + previous_statement_search_text = statement.search_text + + statements_from_file.append(statement) + + if statements_from_file: + self.chatbot.storage.create_many(statements_from_file) + + print("Training took", time.time() - start_time, "seconds.") + + def train(self, *args, **kwargs): + log.error("See asynctrain instead") class TwitterCorpusTrainer(Trainer): @@ -46,4 +186,4 @@ class TwitterCorpusTrainer(Trainer): # # statements_to_create.append(statement) # - # self.chatbot.storage.create_many(statements_to_create) \ No newline at end of file + # self.chatbot.storage.create_many(statements_to_create) From 0e034d83efa7fa66617308774c9d22b3b6cd281b Mon Sep 17 00:00:00 2001 From: bobloy Date: Mon, 15 Feb 2021 10:37:32 -0500 Subject: [PATCH 072/133] Bad idea to steal these, set the empty by default instead --- fifo/task.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/fifo/task.py b/fifo/task.py index cd7200d..e1b7207 100644 --- a/fifo/task.py +++ b/fifo/task.py @@ -75,8 +75,8 @@ things_for_fakemessage_to_steal = [ "_state", "id", "webhook_id", - "reactions", - "attachments", + # "reactions", + # "attachments", "embeds", "application", "activity", @@ -92,10 +92,16 @@ things_for_fakemessage_to_steal = [ "reference", ] +things_fakemessage_sets_by_default = { + "attachments": [], + "reactions": [], +} + class FakeMessage(discord.Message): def __init__(self, *args, message: discord.Message, **kwargs): d = {k: getattr(message, k, None) for k in things_for_fakemessage_to_steal} + d.update(things_fakemessage_sets_by_default) for k, v in d.items(): try: # log.debug(f"{k=} {v=}") From 6363f5eadc833a4a442fff2572d7c4e97bf14a2d Mon Sep 17 00:00:00 2001 From: Obi-Wan3 <44986166+Obi-Wan3@users.noreply.github.com> Date: Tue, 16 Feb 2021 17:15:25 -0800 Subject: [PATCH 073/133] [StealEmoji] update to use guild.emoji_limit --- stealemoji/stealemoji.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/stealemoji/stealemoji.py b/stealemoji/stealemoji.py index a492527..5d3701f 100644 --- a/stealemoji/stealemoji.py +++ b/stealemoji/stealemoji.py @@ -16,16 +16,16 @@ log = logging.getLogger("red.fox_v3.stealemoji") async def check_guild(guild, emoji): - if len(guild.emojis) >= 100: + if len(guild.emojis) >= 2*guild.emoji_limit: return False if len(guild.emojis) < 50: return True if emoji.animated: - return sum(e.animated for e in guild.emojis) < 50 + return sum(e.animated for e in guild.emojis) < guild.emoji_limit else: - return sum(not e.animated for e in guild.emojis) < 50 + return sum(not e.animated for e in guild.emojis) < guild.emoji_limit class StealEmoji(Cog): From 7ad6b156419c147b2e7db692dfc796c90add0320 Mon Sep 17 00:00:00 2001 From: Obi-Wan3 <44986166+Obi-Wan3@users.noreply.github.com> Date: Tue, 16 Feb 2021 17:19:41 -0800 Subject: [PATCH 074/133] Edit to satisfy style requirement --- stealemoji/stealemoji.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stealemoji/stealemoji.py b/stealemoji/stealemoji.py index 5d3701f..de65728 100644 --- a/stealemoji/stealemoji.py +++ b/stealemoji/stealemoji.py @@ -16,7 +16,7 @@ log = logging.getLogger("red.fox_v3.stealemoji") async def check_guild(guild, emoji): - if len(guild.emojis) >= 2*guild.emoji_limit: + if len(guild.emojis) >= 2 * guild.emoji_limit: return False if len(guild.emojis) < 50: From 92957bcb1f87957e2454350d5a1ce85922d37f57 Mon Sep 17 00:00:00 2001 From: Obi-Wan3 <44986166+Obi-Wan3@users.noreply.github.com> Date: Thu, 18 Feb 2021 10:08:24 -0800 Subject: [PATCH 075/133] implement same logic for skipping further checks Co-authored-by: bobloy --- stealemoji/stealemoji.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stealemoji/stealemoji.py b/stealemoji/stealemoji.py index de65728..8f32d74 100644 --- a/stealemoji/stealemoji.py +++ b/stealemoji/stealemoji.py @@ -19,7 +19,7 @@ async def check_guild(guild, emoji): if len(guild.emojis) >= 2 * guild.emoji_limit: return False - if len(guild.emojis) < 50: + if len(guild.emojis) < guild.emoji_limit: return True if emoji.animated: From ee8f6bbf5726dc2f545a2c2c59a754b205099ba4 Mon Sep 17 00:00:00 2001 From: bobloy Date: Fri, 19 Feb 2021 11:07:34 -0500 Subject: [PATCH 076/133] Fix docstrings --- flag/flag.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/flag/flag.py b/flag/flag.py index 6216f65..10f0334 100644 --- a/flag/flag.py +++ b/flag/flag.py @@ -53,9 +53,7 @@ class Flag(Cog): @commands.group() async def flagset(self, ctx: commands.Context): """ - My custom cog - - Extra information goes here + Commands for managing Flag settings """ if ctx.invoked_subcommand is None: pass From 221ca4074bd6f7ec7ae561e5935fce6d4d6b63cd Mon Sep 17 00:00:00 2001 From: bobloy Date: Wed, 24 Feb 2021 12:27:50 -0500 Subject: [PATCH 077/133] Update launchlib to version 2 --- launchlib/info.json | 2 +- launchlib/launchlib.py | 95 +++++++++++++++++++++++++++++++++++++----- 2 files changed, 86 insertions(+), 11 deletions(-) diff --git a/launchlib/info.json b/launchlib/info.json index c1c7ad7..f9b7f11 100644 --- a/launchlib/info.json +++ b/launchlib/info.json @@ -8,7 +8,7 @@ "install_msg": "Thank you for installing LaunchLib. Get started with `[p]load launchlib`, then `[p]help LaunchLib`", "short": "Access launch data for space flights", "end_user_data_statement": "This cog does not store any End User Data", - "requirements": ["python-launch-library>=1.0.6"], + "requirements": ["python-launch-library>=2.0.3"], "tags": [ "bobloy", "utils", diff --git a/launchlib/launchlib.py b/launchlib/launchlib.py index ae870fd..4994a1d 100644 --- a/launchlib/launchlib.py +++ b/launchlib/launchlib.py @@ -1,7 +1,7 @@ import asyncio import functools import logging - +import re import discord import launchlibrary as ll from redbot.core import Config, commands @@ -14,9 +14,7 @@ log = logging.getLogger("red.fox_v3.launchlib") class LaunchLib(commands.Cog): """ - Cog Description - - Less important information about the cog + Cog using `thespacedevs` API to get details about rocket launches """ def __init__(self, bot: Red): @@ -37,27 +35,86 @@ class LaunchLib(commands.Cog): return async def _embed_launch_data(self, launch: ll.AsyncLaunch): - status: ll.AsyncLaunchStatus = await launch.get_status() + + if False: + example_launch = ll.AsyncLaunch( + id="9279744e-46b2-4eca-adea-f1379672ec81", + name="Atlas LV-3A | Samos 2", + tbddate=False, + tbdtime=False, + status={"id": 3, "name": "Success"}, + inhold=False, + windowstart="1961-01-31 20:21:19+00:00", + windowend="1961-01-31 20:21:19+00:00", + net="1961-01-31 20:21:19+00:00", + info_urls=[], + vid_urls=[], + holdreason=None, + failreason=None, + probability=0, + hashtag=None, + agency=None, + changed=None, + pad=ll.Pad( + id=93, + name="Space Launch Complex 3W", + latitude=34.644, + longitude=-120.593, + map_url="http://maps.google.com/maps?q=34.644+N,+120.593+W", + retired=None, + total_launch_count=3, + agency_id=161, + wiki_url=None, + info_url=None, + location=ll.Location( + id=11, + name="Vandenberg AFB, CA, USA", + country_code="USA", + total_launch_count=83, + total_landing_count=3, + pads=None, + ), + map_image="https://spacelaunchnow-prod-east.nyc3.digitaloceanspaces.com/media/launch_images/pad_93_20200803143225.jpg", + ), + rocket=ll.Rocket( + id=2362, + name=None, + default_pads=None, + family=None, + wiki_url=None, + info_url=None, + image_url=None, + ), + missions=None, + ) + + # status: ll.AsyncLaunchStatus = await launch.get_status() + status = launch.status rocket: ll.AsyncRocket = launch.rocket title = launch.name - description = status.description + description = status["name"] urls = launch.vid_urls + launch.info_urls - if not urls and rocket: - urls = rocket.info_urls + [rocket.wiki_url] + if rocket: + urls += [rocket.info_url, rocket.wiki_url] + if launch.pad: + urls += [launch.pad.info_url, launch.pad.wiki_url] + if urls: - url = urls[0] + url = next((url for url in urls if urls is not None), None) else: url = None - color = discord.Color.green() if status.id in [1, 3] else discord.Color.red() + color = discord.Color.green() if status["id"] in [1, 3] else discord.Color.red() em = discord.Embed(title=title, description=description, url=url, color=color) if rocket and rocket.image_url and rocket.image_url != "Array": em.set_image(url=rocket.image_url) + elif launch.pad and launch.pad.map_image: + em.set_image(url=launch.pad.map_image) agency = getattr(launch, "agency", None) if agency is not None: @@ -89,6 +146,16 @@ class LaunchLib(commands.Cog): data = mission.get(f[0], None) if data is not None and data: em.add_field(name=f[1], value=data) + if launch.pad: + location_url = getattr(launch.pad, "map_url", None) + pad_name = getattr(launch.pad, "name", None) + + if pad_name is not None: + if location_url is not None: + location_url = re.sub("[^a-zA-Z0-9/:.'+\"°?=,-]", "", location_url) # Fix bad URLS + em.add_field(name="Launch Pad Name", value=f"[{pad_name}]({location_url})") + else: + em.add_field(name="Launch Pad Name", value=pad_name) if rocket and rocket.family: em.add_field(name="Rocket Family", value=rocket.family) @@ -101,11 +168,17 @@ class LaunchLib(commands.Cog): @commands.group() async def launchlib(self, ctx: commands.Context): + """Base command for getting launches""" if ctx.invoked_subcommand is None: pass @launchlib.command() async def next(self, ctx: commands.Context, num_launches: int = 1): + """ + Show the next launches + + Use `num_launches` to get more than one. + """ # launches = await api.async_next_launches(num_launches) # loop = asyncio.get_running_loop() # @@ -115,6 +188,8 @@ class LaunchLib(commands.Cog): # launches = await self.api.async_fetch_launch(num=num_launches) + # log.debug(str(launches)) + async with ctx.typing(): for x, launch in enumerate(launches): if x >= num_launches: From a5eda8ca2a25e61d5ced5cf2223e1cb1b31e0cf9 Mon Sep 17 00:00:00 2001 From: bobloy Date: Tue, 2 Mar 2021 08:53:50 -0500 Subject: [PATCH 078/133] Forgot how to use regex apparently --- isitdown/isitdown.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/isitdown/isitdown.py b/isitdown/isitdown.py index f786928..b72549a 100644 --- a/isitdown/isitdown.py +++ b/isitdown/isitdown.py @@ -10,9 +10,9 @@ log = logging.getLogger("red.fox_v3.isitdown") class IsItDown(commands.Cog): """ - Cog Description + Cog for checking whether a website is down or not. - Less important information about the cog + Uses the `isitdown.site` API """ def __init__(self, bot: Red): @@ -36,23 +36,25 @@ class IsItDown(commands.Cog): Alias: iid """ try: - resp = await self._check_if_down(url_to_check) + resp, url = await self._check_if_down(url_to_check) except AssertionError: await ctx.maybe_send_embed("Invalid URL provided. Make sure not to include `http://`") return + # log.debug(resp) if resp["isitdown"]: - await ctx.maybe_send_embed(f"{url_to_check} is DOWN!") + await ctx.maybe_send_embed(f"{url} is DOWN!") else: - await ctx.maybe_send_embed(f"{url_to_check} is UP!") + await ctx.maybe_send_embed(f"{url} is UP!") async def _check_if_down(self, url_to_check): - url = re.compile(r"https?://(www\.)?") - url.sub("", url_to_check).strip().strip("/") + re_compiled = re.compile(r"https?://(www\.)?") + url = re_compiled.sub("", url_to_check).strip().strip("/") url = f"https://isitdown.site/api/v3/{url}" + # log.debug(url) async with aiohttp.ClientSession() as session: async with session.get(url) as response: assert response.status == 200 resp = await response.json() - return resp + return resp, url From bf9115e13cf81279aeff4ce61255d2dbe6483992 Mon Sep 17 00:00:00 2001 From: bobloy Date: Wed, 10 Mar 2021 13:36:37 -0500 Subject: [PATCH 079/133] Update for new channel --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ec76ead..b1c3b73 100644 --- a/README.md +++ b/README.md @@ -53,7 +53,7 @@ Check out *Deprecated* my V2 cogs at [Fox-Cogs v2](https://github.com/bobloy/Fox # Contact Get support on the [Third Party Cog Server](https://discord.gg/GET4DVk) -Feel free to @ me in the #support_othercogs channel +Feel free to @ me in the #support_fox-v3 channel Discord: Bobloy#6513 From 5a26b48fdacf6c360da4acddbedd081c8d171b74 Mon Sep 17 00:00:00 2001 From: Antoine Rybacki Date: Fri, 12 Mar 2021 23:57:56 +0100 Subject: [PATCH 080/133] [Chatter] Allow bot to reply to maintain conversation continuity --- chatter/chat.py | 34 ++++++++++++++++++++++++++++++++-- chatter/info.json | 2 +- 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/chatter/chat.py b/chatter/chat.py index 41affb6..84e8fbb 100644 --- a/chatter/chat.py +++ b/chatter/chat.py @@ -53,7 +53,13 @@ class Chatter(Cog): self.bot = bot self.config = Config.get_conf(self, identifier=6710497116116101114) default_global = {} - default_guild = {"whitelist": None, "days": 1, "convo_delta": 15, "chatchannel": None} + default_guild = { + "whitelist": None, + "days": 1, + "convo_delta": 15, + "chatchannel": None, + "reply": False, + } path: pathlib.Path = cog_data_path(self) self.data_path = path / "database.sqlite3" @@ -213,6 +219,25 @@ class Chatter(Cog): await self.config.guild(ctx.guild).chatchannel.set(channel.id) await ctx.maybe_send_embed(f"Chat channel is now {channel.mention}") + @checks.admin() + @chatter.command(name="reply") + async def chatter_reply(self, ctx: commands.Context, toggle: Optional[bool] = None): + """ + Toggle bot reply to messages if conversation continuity is not present + + """ + reply = await self.config.guild(ctx.guild).reply() + if toggle is None: + toggle = not reply + await self.config.guild(ctx.guild).reply.set(toggle) + + if toggle: + await ctx.send("I will now respond to you if conversation continuity is not present") + else: + await ctx.send( + "I will not reply to your message if conversation continuity is not present, anymore" + ) + @checks.is_owner() @chatter.command(name="cleardata") async def chatter_cleardata(self, ctx: commands.Context, confirm: bool = False): @@ -493,7 +518,12 @@ class Chatter(Cog): async with channel.typing(): future = await self.loop.run_in_executor(None, self.chatbot.get_response, text) + replying = None + if await self.config.guild(guild).reply(): + if message != ctx.channel.last_message: + replying = message + if future and str(future): - await channel.send(str(future)) + await channel.send(str(future), reference=replying) else: await channel.send(":thinking:") diff --git a/chatter/info.json b/chatter/info.json index 85107ed..a3fe0da 100644 --- a/chatter/info.json +++ b/chatter/info.json @@ -2,7 +2,7 @@ "author": [ "Bobloy" ], - "min_bot_version": "3.4.0", + "min_bot_version": "3.4.6", "description": "Create an offline chatbot that talks like your average member using Machine Learning. See setup instructions at https://github.com/bobloy/Fox-V3/tree/master/chatter", "hidden": false, "install_msg": "Thank you for installing Chatter! Please make sure you check the install instructions at https://github.com/bobloy/Fox-V3/blob/master/chatter/README.md\nAfter that, get started ith `[p]load chatter` and `[p]help Chatter`", From cc199c395d7851614abdda469382760c75491654 Mon Sep 17 00:00:00 2001 From: bobloy Date: Mon, 15 Mar 2021 08:16:20 -0400 Subject: [PATCH 081/133] Discord file now takes a path, so just give em a path. --- qrinvite/qrinvite.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/qrinvite/qrinvite.py b/qrinvite/qrinvite.py index ab5f5dc..a91f7ab 100644 --- a/qrinvite/qrinvite.py +++ b/qrinvite/qrinvite.py @@ -96,8 +96,8 @@ class QRInvite(Cog): ) png_path: pathlib.Path = path / (ctx.guild.icon + "_qrcode.png") - with png_path.open("rb") as png_fp: - await ctx.send(file=discord.File(png_fp.read(), "qrcode.png")) + # with png_path.open("rb") as png_fp: + await ctx.send(file=discord.File(png_path, "qrcode.png")) def convert_webp_to_png(path): From f7dad0aa3f9d0e5a73778062314ba8621d490a4a Mon Sep 17 00:00:00 2001 From: Antoine Rybacki Date: Mon, 15 Mar 2021 14:25:46 +0100 Subject: [PATCH 082/133] [Chatter] Bot will respond to reply --- chatter/chat.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/chatter/chat.py b/chatter/chat.py index 84e8fbb..ed8b49d 100644 --- a/chatter/chat.py +++ b/chatter/chat.py @@ -500,7 +500,15 @@ class Chatter(Cog): # Thank you Cog-Creators channel: discord.TextChannel = message.channel - if guild is not None and channel.id == await self.config.guild(guild).chatchannel(): + # is_reply = False # this is only useful with in_response_to + if ( + message.reference is not None + and isinstance(message.reference.resolved,discord.Message) + and message.reference.resolved.author.id == self.bot.user.id + ): + # is_reply = True # this is only useful with in_response_to + pass # this is a reply to the bot, good to go + elif guild is not None and channel.id == await self.config.guild(guild).chatchannel(): pass # good to go else: when_mentionables = commands.when_mentioned(self.bot, message) From 42bdc640289d82318229ca3de0b8cff3dfe070dc Mon Sep 17 00:00:00 2001 From: Antoine Rybacki Date: Mon, 15 Mar 2021 14:29:34 +0100 Subject: [PATCH 083/133] Black format fix --- chatter/chat.py | 2 +- launchlib/launchlib.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/chatter/chat.py b/chatter/chat.py index ed8b49d..971492c 100644 --- a/chatter/chat.py +++ b/chatter/chat.py @@ -503,7 +503,7 @@ class Chatter(Cog): # is_reply = False # this is only useful with in_response_to if ( message.reference is not None - and isinstance(message.reference.resolved,discord.Message) + and isinstance(message.reference.resolved, discord.Message) and message.reference.resolved.author.id == self.bot.user.id ): # is_reply = True # this is only useful with in_response_to diff --git a/launchlib/launchlib.py b/launchlib/launchlib.py index 4994a1d..3d3eb0e 100644 --- a/launchlib/launchlib.py +++ b/launchlib/launchlib.py @@ -152,7 +152,9 @@ class LaunchLib(commands.Cog): if pad_name is not None: if location_url is not None: - location_url = re.sub("[^a-zA-Z0-9/:.'+\"°?=,-]", "", location_url) # Fix bad URLS + location_url = re.sub( + "[^a-zA-Z0-9/:.'+\"°?=,-]", "", location_url + ) # Fix bad URLS em.add_field(name="Launch Pad Name", value=f"[{pad_name}]({location_url})") else: em.add_field(name="Launch Pad Name", value=pad_name) From 920f8817d75c26df7f6042e181c3ddcab7702b49 Mon Sep 17 00:00:00 2001 From: bobloy Date: Mon, 15 Mar 2021 10:15:13 -0400 Subject: [PATCH 084/133] Use guild.id and author.id for file name, support using jpg --- qrinvite/qrinvite.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/qrinvite/qrinvite.py b/qrinvite/qrinvite.py index a91f7ab..a4c377c 100644 --- a/qrinvite/qrinvite.py +++ b/qrinvite/qrinvite.py @@ -68,7 +68,7 @@ class QRInvite(Cog): extension = pathlib.Path(image_url).parts[-1].replace(".", "?").split("?")[1] path: pathlib.Path = cog_data_path(self) - image_path = path / (ctx.guild.icon + "." + extension) + image_path = path / f"{ctx.guild.id}-{ctx.author.id}.{extension}" async with aiohttp.ClientSession() as session: async with session.get(image_url) as response: image = await response.read() @@ -83,6 +83,8 @@ class QRInvite(Cog): return elif extension == "png": new_path = str(image_path) + elif extension == "jpg": + new_path = convert_jpg_to_png(str(image_path)) else: await ctx.maybe_send_embed(f"{extension} is not supported yet, stay tuned") return @@ -110,3 +112,10 @@ def convert_webp_to_png(path): new_path = path.replace(".webp", ".png") im.save(new_path, transparency=255) return new_path + + +def convert_jpg_to_png(path): + im = Image.open(path) + new_path = path.replace(".jpg", ".png") + im.save(new_path) + return new_path From 578ea4a555edbdd9d30a4221ed9ac145e1bd129f Mon Sep 17 00:00:00 2001 From: bobloy Date: Mon, 15 Mar 2021 10:31:29 -0400 Subject: [PATCH 085/133] Consistently avoid guild.icon --- qrinvite/qrinvite.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/qrinvite/qrinvite.py b/qrinvite/qrinvite.py index a4c377c..684b69d 100644 --- a/qrinvite/qrinvite.py +++ b/qrinvite/qrinvite.py @@ -67,8 +67,10 @@ class QRInvite(Cog): extension = pathlib.Path(image_url).parts[-1].replace(".", "?").split("?")[1] + save_as_name = f"{ctx.guild.id}-{ctx.author.id}" + path: pathlib.Path = cog_data_path(self) - image_path = path / f"{ctx.guild.id}-{ctx.author.id}.{extension}" + image_path = path / f"{save_as_name}.{extension}" async with aiohttp.ClientSession() as session: async with session.get(image_url) as response: image = await response.read() @@ -77,27 +79,27 @@ class QRInvite(Cog): file.write(image) if extension == "webp": - new_path = convert_webp_to_png(str(image_path)) + new_image_path = convert_webp_to_png(str(image_path)) elif extension == "gif": await ctx.maybe_send_embed("gif is not supported yet, stay tuned") return elif extension == "png": - new_path = str(image_path) + new_image_path = str(image_path) elif extension == "jpg": - new_path = convert_jpg_to_png(str(image_path)) + new_image_path = convert_jpg_to_png(str(image_path)) else: await ctx.maybe_send_embed(f"{extension} is not supported yet, stay tuned") return myqr.run( invite, - picture=new_path, - save_name=ctx.guild.icon + "_qrcode.png", + picture=new_image_path, + save_name=f"{save_as_name}_qrcode.png", save_dir=str(cog_data_path(self)), colorized=colorized, ) - png_path: pathlib.Path = path / (ctx.guild.icon + "_qrcode.png") + png_path: pathlib.Path = path / f"{save_as_name}_qrcode.png" # with png_path.open("rb") as png_fp: await ctx.send(file=discord.File(png_path, "qrcode.png")) From d32de1586ffb35c3b052d53044fdd89773ef41dc Mon Sep 17 00:00:00 2001 From: bobloy Date: Mon, 15 Mar 2021 15:40:25 -0400 Subject: [PATCH 086/133] Reply enabled by default cause it's cool. --- chatter/chat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chatter/chat.py b/chatter/chat.py index 971492c..1419bbf 100644 --- a/chatter/chat.py +++ b/chatter/chat.py @@ -58,7 +58,7 @@ class Chatter(Cog): "days": 1, "convo_delta": 15, "chatchannel": None, - "reply": False, + "reply": True, } path: pathlib.Path = cog_data_path(self) self.data_path = path / "database.sqlite3" From 8acbc5d9645e1e23e65d60eb00e929d202c4a3e5 Mon Sep 17 00:00:00 2001 From: bobloy Date: Mon, 15 Mar 2021 15:48:34 -0400 Subject: [PATCH 087/133] Whatever this commit is --- chatter/chat.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/chatter/chat.py b/chatter/chat.py index e29c317..500284c 100644 --- a/chatter/chat.py +++ b/chatter/chat.py @@ -620,7 +620,9 @@ class Chatter(Cog): replying = message if future and str(future): - self._last_message_per_channel[ctx.channel.id] = await channel.send(str(future), reference=replying) + self._last_message_per_channel[ctx.channel.id] = await channel.send( + str(future), reference=replying + ) else: await ctx.send(":thinking:") From 7811c71edbcb7b059e0181fc87cfb3934ba95c53 Mon Sep 17 00:00:00 2001 From: bobloy Date: Tue, 16 Mar 2021 16:00:42 -0400 Subject: [PATCH 088/133] Use is_reply to train --- chatter/chat.py | 37 ++++++++++++++++++++++++------------- 1 file changed, 24 insertions(+), 13 deletions(-) diff --git a/chatter/chat.py b/chatter/chat.py index 500284c..81d09a8 100644 --- a/chatter/chat.py +++ b/chatter/chat.py @@ -564,13 +564,13 @@ class Chatter(Cog): # Thank you Cog-Creators channel: discord.TextChannel = message.channel - # is_reply = False # this is only useful with in_response_to + is_reply = False # this is only useful with in_response_to if ( message.reference is not None and isinstance(message.reference.resolved, discord.Message) and message.reference.resolved.author.id == self.bot.user.id ): - # is_reply = True # this is only useful with in_response_to + is_reply = True # this is only useful with in_response_to pass # this is a reply to the bot, good to go elif guild is not None and channel.id == await self.config.guild(guild).chatchannel(): pass # good to go @@ -592,7 +592,9 @@ class Chatter(Cog): if not self._guild_cache[ctx.guild.id]: self._guild_cache[ctx.guild.id] = await self.config.guild(ctx.guild).all() - if self._last_message_per_channel[ctx.channel.id] is not None: + if is_reply: + in_response_to = message.reference.resolved.content + elif self._last_message_per_channel[ctx.channel.id] is not None: last_m: discord.Message = self._last_message_per_channel[ctx.channel.id] minutes = self._guild_cache[ctx.guild.id]["convo_delta"] if (datetime.utcnow() - last_m.created_at).seconds > minutes * 60: @@ -602,16 +604,25 @@ class Chatter(Cog): else: in_response_to = None - if in_response_to is None: - log.debug("Generating response") - Statement = self.chatbot.storage.get_object("statement") - future = await self.loop.run_in_executor( - None, self.chatbot.generate_response, Statement(text) - ) - else: - log.debug("Getting response") - future = await self.loop.run_in_executor( - None, partial(self.chatbot.get_response, text, in_response_to=in_response_to) + # Always use generate reponse + # Chatterbot tries to learn based on the result it comes up with, which is dumb + log.debug("Generating response") + Statement = self.chatbot.storage.get_object("statement") + future = await self.loop.run_in_executor( + None, self.chatbot.generate_response, Statement(text) + ) + + if in_response_to is not None: + log.debug("learning response") + learning_task = asyncio.create_task( + self.loop.run_in_executor( + None, + partial( + self.chatbot.learn_response, + Statement(text), + previous_statement=in_response_to, + ), + ) ) replying = None From 0475b18437845b2de1db44062340e9706d687b1c Mon Sep 17 00:00:00 2001 From: Sourcery AI Date: Thu, 18 Mar 2021 17:20:04 +0000 Subject: [PATCH 089/133] 'Refactored by Sourcery' --- announcedaily/announcedaily.py | 3 +- audiotrivia/audiotrivia.py | 2 +- ccrole/ccrole.py | 12 ++-- chatter/chat.py | 3 +- coglint/coglint.py | 6 +- conquest/conquest.py | 10 ++- conquest/mapmaker.py | 3 +- conquest/regioner.py | 2 +- exclusiverole/exclusiverole.py | 7 +- fifo/fifo.py | 15 ++--- flag/flag.py | 3 +- hangman/hangman.py | 9 +-- infochannel/infochannel.py | 10 +-- launchlib/launchlib.py | 61 +---------------- leaver/leaver.py | 5 +- lseen/lseen.py | 6 +- planttycoon/planttycoon.py | 115 +++++++++++++++------------------ reactrestrict/reactrestrict.py | 8 ++- rpsls/rpsls.py | 11 ++-- stealemoji/stealemoji.py | 60 +++++++++-------- timerole/timerole.py | 7 +- unicode/unicode.py | 3 +- werewolf/builder.py | 9 +-- werewolf/game.py | 26 ++++---- werewolf/werewolf.py | 9 +-- 25 files changed, 149 insertions(+), 256 deletions(-) diff --git a/announcedaily/announcedaily.py b/announcedaily/announcedaily.py index aa50e6c..98690f7 100644 --- a/announcedaily/announcedaily.py +++ b/announcedaily/announcedaily.py @@ -54,8 +54,7 @@ class AnnounceDaily(Cog): Do `[p]help annd ` for more details """ - if ctx.invoked_subcommand is None: - pass + pass @commands.command() @checks.guildowner() diff --git a/audiotrivia/audiotrivia.py b/audiotrivia/audiotrivia.py index 9617f32..73eca95 100644 --- a/audiotrivia/audiotrivia.py +++ b/audiotrivia/audiotrivia.py @@ -168,7 +168,7 @@ class AudioTrivia(Trivia): @commands.guild_only() async def audiotrivia_list(self, ctx: commands.Context): """List available trivia including audio categories.""" - lists = set(p.stem for p in self._all_audio_lists()) + lists = {p.stem for p in self._all_audio_lists()} if await ctx.embed_requested(): await ctx.send( embed=discord.Embed( diff --git a/ccrole/ccrole.py b/ccrole/ccrole.py index 5248766..e3691f8 100644 --- a/ccrole/ccrole.py +++ b/ccrole/ccrole.py @@ -48,8 +48,7 @@ class CCRole(commands.Cog): """Custom commands management with roles Highly customizable custom commands with role management.""" - if not ctx.invoked_subcommand: - pass + pass @ccrole.command(name="add") @checks.mod_or_permissions(administrator=True) @@ -228,7 +227,8 @@ class CCRole(commands.Cog): if not role_list: return "None" return ", ".join( - [discord.utils.get(ctx.guild.roles, id=roleid).name for roleid in role_list] + discord.utils.get(ctx.guild.roles, id=roleid).name + for roleid in role_list ) embed.add_field(name="Text", value="```{}```".format(cmd["text"]), inline=False) @@ -252,7 +252,7 @@ class CCRole(commands.Cog): ) return - cmd_list = ", ".join([ctx.prefix + c for c in sorted(cmd_list.keys())]) + cmd_list = ", ".join(ctx.prefix + c for c in sorted(cmd_list.keys())) cmd_list = "Custom commands:\n\n" + cmd_list if ( @@ -325,8 +325,8 @@ class CCRole(commands.Cog): async def eval_cc(self, cmd, message: discord.Message, ctx: commands.Context): """Does all the work""" - if cmd["proles"] and not ( - set(role.id for role in message.author.roles) & set(cmd["proles"]) + if cmd["proles"] and not {role.id for role in message.author.roles} & set( + cmd["proles"] ): log.debug(f"{message.author} missing required role to execute {ctx.invoked_with}") return # Not authorized, do nothing diff --git a/chatter/chat.py b/chatter/chat.py index 1419bbf..de0e20a 100644 --- a/chatter/chat.py +++ b/chatter/chat.py @@ -196,8 +196,7 @@ class Chatter(Cog): """ Base command for this cog. Check help for the commands list. """ - if ctx.invoked_subcommand is None: - pass + pass @checks.admin() @chatter.command(name="channel") diff --git a/coglint/coglint.py b/coglint/coglint.py index 6595980..9c4739c 100644 --- a/coglint/coglint.py +++ b/coglint/coglint.py @@ -58,11 +58,7 @@ class CogLint(Cog): future = await self.bot.loop.run_in_executor(None, lint.py_run, path, "return_std=True") - if future: - (pylint_stdout, pylint_stderr) = future - else: - (pylint_stdout, pylint_stderr) = None, None - + (pylint_stdout, pylint_stderr) = future or (None, None) # print(pylint_stderr) # print(pylint_stdout) diff --git a/conquest/conquest.py b/conquest/conquest.py index fdf5e96..fa70911 100644 --- a/conquest/conquest.py +++ b/conquest/conquest.py @@ -67,9 +67,8 @@ class Conquest(commands.Cog): """ Base command for conquest cog. Start with `[p]conquest set map` to select a map. """ - if ctx.invoked_subcommand is None: - if self.current_map is not None: - await self._conquest_current(ctx) + if ctx.invoked_subcommand is None and self.current_map is not None: + await self._conquest_current(ctx) @conquest.command(name="list") async def _conquest_list(self, ctx: commands.Context): @@ -80,14 +79,13 @@ class Conquest(commands.Cog): with maps_json.open() as maps: maps_json = json.load(maps) - map_list = "\n".join(map_name for map_name in maps_json["maps"]) + map_list = "\n".join(maps_json["maps"]) await ctx.maybe_send_embed(f"Current maps:\n{map_list}") @conquest.group(name="set") async def conquest_set(self, ctx: commands.Context): """Base command for admin actions like selecting a map""" - if ctx.invoked_subcommand is None: - pass + pass @conquest_set.command(name="resetzoom") async def _conquest_set_resetzoom(self, ctx: commands.Context): diff --git a/conquest/mapmaker.py b/conquest/mapmaker.py index 0cde96a..5fd90b2 100644 --- a/conquest/mapmaker.py +++ b/conquest/mapmaker.py @@ -30,8 +30,7 @@ class MapMaker(commands.Cog): """ Base command for managing current maps or creating new ones """ - if ctx.invoked_subcommand is None: - pass + pass @mapmaker.command(name="upload") async def _mapmaker_upload(self, ctx: commands.Context, map_path=""): diff --git a/conquest/regioner.py b/conquest/regioner.py index dc77373..b89bc5f 100644 --- a/conquest/regioner.py +++ b/conquest/regioner.py @@ -65,7 +65,7 @@ def floodfill(image, xy, value, border=None, thresh=0) -> set: if border is None: fill = _color_diff(p, background) <= thresh else: - fill = p != value and p != border + fill = p not in [value, border] if fill: pixel[s, t] = value new_edge.add((s, t)) diff --git a/exclusiverole/exclusiverole.py b/exclusiverole/exclusiverole.py index 19635b2..63b7460 100644 --- a/exclusiverole/exclusiverole.py +++ b/exclusiverole/exclusiverole.py @@ -27,8 +27,7 @@ class ExclusiveRole(Cog): async def exclusive(self, ctx): """Base command for managing exclusive roles""" - if not ctx.invoked_subcommand: - pass + pass @exclusive.command(name="add") @checks.mod_or_permissions(administrator=True) @@ -85,7 +84,7 @@ class ExclusiveRole(Cog): if role_set is None: role_set = set(await self.config.guild(member.guild).role_list()) - member_set = set([role.id for role in member.roles]) + member_set = {role.id for role in member.roles} to_remove = (member_set - role_set) - {member.guild.default_role.id} if to_remove and member_set & role_set: @@ -103,7 +102,7 @@ class ExclusiveRole(Cog): await asyncio.sleep(1) role_set = set(await self.config.guild(after.guild).role_list()) - member_set = set([role.id for role in after.roles]) + member_set = {role.id for role in after.roles} if role_set & member_set: try: diff --git a/fifo/fifo.py b/fifo/fifo.py index d152609..24d01f3 100644 --- a/fifo/fifo.py +++ b/fifo/fifo.py @@ -68,10 +68,7 @@ class CapturePrint: self.string = None def write(self, string): - if self.string is None: - self.string = string - else: - self.string = self.string + "\n" + string + self.string = string if self.string is None else self.string + "\n" + string class FIFO(commands.Cog): @@ -197,8 +194,8 @@ class FIFO(commands.Cog): async def _get_tz(self, user: Union[discord.User, discord.Member]) -> Union[None, tzinfo]: if self.tz_cog is None: self.tz_cog = self.bot.get_cog("Timezone") - if self.tz_cog is None: - self.tz_cog = False # only try once to get the timezone cog + if self.tz_cog is None: + self.tz_cog = False # only try once to get the timezone cog if not self.tz_cog: return None @@ -230,8 +227,7 @@ class FIFO(commands.Cog): """ Base command for handling scheduling of tasks """ - if ctx.invoked_subcommand is None: - pass + pass @fifo.command(name="wakeup") async def fifo_wakeup(self, ctx: commands.Context): @@ -522,8 +518,7 @@ class FIFO(commands.Cog): """ Add a new trigger for a task from the current guild. """ - if ctx.invoked_subcommand is None: - pass + pass @fifo_trigger.command(name="interval") async def fifo_trigger_interval( diff --git a/flag/flag.py b/flag/flag.py index 10f0334..e267297 100644 --- a/flag/flag.py +++ b/flag/flag.py @@ -55,8 +55,7 @@ class Flag(Cog): """ Commands for managing Flag settings """ - if ctx.invoked_subcommand is None: - pass + pass @flagset.command(name="expire") async def flagset_expire(self, ctx: commands.Context, days: int): diff --git a/hangman/hangman.py b/hangman/hangman.py index 2b6ab07..d737aea 100644 --- a/hangman/hangman.py +++ b/hangman/hangman.py @@ -147,8 +147,7 @@ class Hangman(Cog): @checks.mod_or_permissions(administrator=True) async def hangset(self, ctx): """Adjust hangman settings""" - if ctx.invoked_subcommand is None: - pass + pass @hangset.command() async def face(self, ctx: commands.Context, theface): @@ -250,7 +249,7 @@ class Hangman(Cog): self.winbool[guild] = True for i in self.the_data[guild]["answer"]: - if i == " " or i == "-": + if i in [" ", "-"]: out_str += i * 2 elif i in self.the_data[guild]["guesses"] or i not in "ABCDEFGHIJKLMNOPQRSTUVWXYZ": out_str += "__" + i + "__ " @@ -262,9 +261,7 @@ class Hangman(Cog): def _guesslist(self, guild): """Returns the current letter list""" - out_str = "" - for i in self.the_data[guild]["guesses"]: - out_str += str(i) + "," + out_str = "".join(str(i) + "," for i in self.the_data[guild]["guesses"]) out_str = out_str[:-1] return out_str diff --git a/infochannel/infochannel.py b/infochannel/infochannel.py index 33e2b10..fe8589f 100644 --- a/infochannel/infochannel.py +++ b/infochannel/infochannel.py @@ -65,9 +65,12 @@ class InfoChannel(Cog): "offline": "Offline: {count}", } - default_channel_ids = {k: None for k in self.default_channel_names.keys()} + default_channel_ids = {k: None for k in self.default_channel_names} # Only members is enabled by default - default_enabled_counts = {k: k == "members" for k in self.default_channel_names.keys()} + default_enabled_counts = { + k: k == "members" for k in self.default_channel_names + } + default_guild = { "category_id": None, @@ -159,8 +162,7 @@ class InfoChannel(Cog): """ Toggle different types of infochannels """ - if not ctx.invoked_subcommand: - pass + pass @infochannelset.command(name="togglechannel") async def _infochannelset_togglechannel( diff --git a/launchlib/launchlib.py b/launchlib/launchlib.py index 3d3eb0e..2a30c3e 100644 --- a/launchlib/launchlib.py +++ b/launchlib/launchlib.py @@ -36,58 +36,6 @@ class LaunchLib(commands.Cog): async def _embed_launch_data(self, launch: ll.AsyncLaunch): - if False: - example_launch = ll.AsyncLaunch( - id="9279744e-46b2-4eca-adea-f1379672ec81", - name="Atlas LV-3A | Samos 2", - tbddate=False, - tbdtime=False, - status={"id": 3, "name": "Success"}, - inhold=False, - windowstart="1961-01-31 20:21:19+00:00", - windowend="1961-01-31 20:21:19+00:00", - net="1961-01-31 20:21:19+00:00", - info_urls=[], - vid_urls=[], - holdreason=None, - failreason=None, - probability=0, - hashtag=None, - agency=None, - changed=None, - pad=ll.Pad( - id=93, - name="Space Launch Complex 3W", - latitude=34.644, - longitude=-120.593, - map_url="http://maps.google.com/maps?q=34.644+N,+120.593+W", - retired=None, - total_launch_count=3, - agency_id=161, - wiki_url=None, - info_url=None, - location=ll.Location( - id=11, - name="Vandenberg AFB, CA, USA", - country_code="USA", - total_launch_count=83, - total_landing_count=3, - pads=None, - ), - map_image="https://spacelaunchnow-prod-east.nyc3.digitaloceanspaces.com/media/launch_images/pad_93_20200803143225.jpg", - ), - rocket=ll.Rocket( - id=2362, - name=None, - default_pads=None, - family=None, - wiki_url=None, - info_url=None, - image_url=None, - ), - missions=None, - ) - # status: ll.AsyncLaunchStatus = await launch.get_status() status = launch.status @@ -102,11 +50,7 @@ class LaunchLib(commands.Cog): if launch.pad: urls += [launch.pad.info_url, launch.pad.wiki_url] - if urls: - url = next((url for url in urls if urls is not None), None) - else: - url = None - + url = next((url for url in urls if urls is not None), None) if urls else None color = discord.Color.green() if status["id"] in [1, 3] else discord.Color.red() em = discord.Embed(title=title, description=description, url=url, color=color) @@ -171,8 +115,7 @@ class LaunchLib(commands.Cog): @commands.group() async def launchlib(self, ctx: commands.Context): """Base command for getting launches""" - if ctx.invoked_subcommand is None: - pass + pass @launchlib.command() async def next(self, ctx: commands.Context, num_launches: int = 1): diff --git a/leaver/leaver.py b/leaver/leaver.py index 0c0d947..76f29f5 100644 --- a/leaver/leaver.py +++ b/leaver/leaver.py @@ -25,8 +25,7 @@ class Leaver(Cog): @checks.mod_or_permissions(administrator=True) async def leaverset(self, ctx): """Adjust leaver settings""" - if ctx.invoked_subcommand is None: - pass + pass @leaverset.command() async def channel(self, ctx: Context): @@ -57,5 +56,3 @@ class Leaver(Cog): ) else: await channel.send(out) - else: - pass diff --git a/lseen/lseen.py b/lseen/lseen.py index 69bcf87..2ddb0e1 100644 --- a/lseen/lseen.py +++ b/lseen/lseen.py @@ -45,14 +45,12 @@ class LastSeen(Cog): @staticmethod def get_date_time(s): - d = dateutil.parser.parse(s) - return d + return dateutil.parser.parse(s) @commands.group(aliases=["setlseen"], name="lseenset") async def lset(self, ctx: commands.Context): """Change settings for lseen""" - if ctx.invoked_subcommand is None: - pass + pass @lset.command(name="toggle") async def lset_toggle(self, ctx: commands.Context): diff --git a/planttycoon/planttycoon.py b/planttycoon/planttycoon.py index 4209b53..54d0119 100644 --- a/planttycoon/planttycoon.py +++ b/planttycoon/planttycoon.py @@ -111,9 +111,8 @@ async def _withdraw_points(gardener: Gardener, amount): if (gardener.points - amount) < 0: return False - else: - gardener.points -= amount - return True + gardener.points -= amount + return True class PlantTycoon(commands.Cog): @@ -245,13 +244,12 @@ class PlantTycoon(commands.Cog): await self._load_plants_products() modifiers = sum( - [ - self.products[product]["modifier"] - for product in gardener.products - if gardener.products[product] > 0 - ] + self.products[product]["modifier"] + for product in gardener.products + if gardener.products[product] > 0 ) + degradation = ( 100 / (gardener.current["time"] / 60) @@ -290,38 +288,33 @@ class PlantTycoon(commands.Cog): product = product.lower() product_category = product_category.lower() if product in self.products and self.products[product]["category"] == product_category: - if product in gardener.products: - if gardener.products[product] > 0: - gardener.current["health"] += self.products[product]["health"] - gardener.products[product] -= 1 - if gardener.products[product] == 0: - del gardener.products[product.lower()] - if product_category == "water": - emoji = ":sweat_drops:" - elif product_category == "fertilizer": - emoji = ":poop:" - # elif product_category == "tool": - else: - emoji = ":scissors:" - message = "Your plant got some health back! {}".format(emoji) - if gardener.current["health"] > gardener.current["threshold"]: - gardener.current["health"] -= self.products[product]["damage"] - if product_category == "tool": - damage_msg = "You used {} too many times!".format(product) - else: - damage_msg = "You gave too much of {}.".format(product) - message = "{} Your plant lost some health. :wilted_rose:".format( - damage_msg - ) - gardener.points += self.defaults["points"]["add_health"] - await gardener.save_gardener() + if product in gardener.products and gardener.products[product] > 0: + gardener.current["health"] += self.products[product]["health"] + gardener.products[product] -= 1 + if gardener.products[product] == 0: + del gardener.products[product.lower()] + if product_category == "fertilizer": + emoji = ":poop:" + elif product_category == "water": + emoji = ":sweat_drops:" else: - message = "You have no {}. Go buy some!".format(product) + emoji = ":scissors:" + message = "Your plant got some health back! {}".format(emoji) + if gardener.current["health"] > gardener.current["threshold"]: + gardener.current["health"] -= self.products[product]["damage"] + if product_category == "tool": + damage_msg = "You used {} too many times!".format(product) + else: + damage_msg = "You gave too much of {}.".format(product) + message = "{} Your plant lost some health. :wilted_rose:".format( + damage_msg + ) + gardener.points += self.defaults["points"]["add_health"] + await gardener.save_gardener() + elif product in gardener.products or product_category != "tool": + message = "You have no {}. Go buy some!".format(product) else: - if product_category == "tool": - message = "You don't have a {}. Go buy one!".format(product) - else: - message = "You have no {}. Go buy some!".format(product) + message = "You don't have a {}. Go buy one!".format(product) else: message = "Are you sure you are using {}?".format(product_category) @@ -412,24 +405,18 @@ class PlantTycoon(commands.Cog): gardener.current = plant await gardener.save_gardener() - em = discord.Embed(description=message, color=discord.Color.green()) else: plant = gardener.current message = "You're already growing {} **{}**, silly.".format( plant["article"], plant["name"] ) - em = discord.Embed(description=message, color=discord.Color.green()) - + em = discord.Embed(description=message, color=discord.Color.green()) await ctx.send(embed=em) @_gardening.command(name="profile") async def _profile(self, ctx: commands.Context, *, member: discord.Member = None): """Check your gardening profile.""" - if member is not None: - author = member - else: - author = ctx.author - + author = member if member is not None else ctx.author gardener = await self._gardener(author) try: await self._apply_degradation(gardener) @@ -440,9 +427,7 @@ class PlantTycoon(commands.Cog): avatar = author.avatar_url if author.avatar else author.default_avatar_url em.set_author(name="Gardening profile of {}".format(author.name), icon_url=avatar) em.add_field(name="**Thneeds**", value=str(gardener.points)) - if not gardener.current: - em.add_field(name="**Currently growing**", value="None") - else: + if gardener.current: em.set_thumbnail(url=gardener.current["image"]) em.add_field( name="**Currently growing**", @@ -450,16 +435,17 @@ class PlantTycoon(commands.Cog): gardener.current["name"], gardener.current["health"] ), ) + else: + em.add_field(name="**Currently growing**", value="None") if not gardener.badges: em.add_field(name="**Badges**", value="None") else: - badges = "" - for badge in gardener.badges: - badges += "{}\n".format(badge.capitalize()) + badges = "".join( + "{}\n".format(badge.capitalize()) for badge in gardener.badges + ) + em.add_field(name="**Badges**", value=badges) - if not gardener.products: - em.add_field(name="**Products**", value="None") - else: + if gardener.products: products = "" for product_name, product_data in gardener.products.items(): if self.products[product_name] is None: @@ -470,6 +456,8 @@ class PlantTycoon(commands.Cog): self.products[product_name]["modifier"], ) em.add_field(name="**Products**", value=products) + else: + em.add_field(name="**Products**", value="None") if gardener.current: degradation = await self._degradation(gardener) die_in = await _die_in(gardener, degradation) @@ -600,7 +588,6 @@ class PlantTycoon(commands.Cog): self.products[pd]["category"], ), ) - await ctx.send(embed=em) else: if amount <= 0: message = "Invalid amount! Must be greater than 1" @@ -629,7 +616,8 @@ class PlantTycoon(commands.Cog): else: message = "I don't have this product." em = discord.Embed(description=message, color=discord.Color.green()) - await ctx.send(embed=em) + + await ctx.send(embed=em) @_gardening.command(name="convert") async def _convert(self, ctx: commands.Context, amount: int): @@ -663,8 +651,7 @@ class PlantTycoon(commands.Cog): else: gardener.current = {} message = "You successfully shovelled your plant out." - if gardener.points < 0: - gardener.points = 0 + gardener.points = max(gardener.points, 0) await gardener.save_gardener() em = discord.Embed(description=message, color=discord.Color.dark_grey()) @@ -681,12 +668,12 @@ class PlantTycoon(commands.Cog): except discord.Forbidden: # Couldn't DM the degradation await ctx.send("ERROR\nYou blocked me, didn't you?") - product = "water" - product_category = "water" if not gardener.current: message = "You're currently not growing a plant." await _send_message(channel, message) else: + product = "water" + product_category = "water" await self._add_health(channel, gardener, product, product_category) @commands.command(name="fertilize") @@ -700,11 +687,11 @@ class PlantTycoon(commands.Cog): await ctx.send("ERROR\nYou blocked me, didn't you?") channel = ctx.channel product = fertilizer - product_category = "fertilizer" if not gardener.current: message = "You're currently not growing a plant." await _send_message(channel, message) else: + product_category = "fertilizer" await self._add_health(channel, gardener, product, product_category) @commands.command(name="prune") @@ -717,12 +704,12 @@ class PlantTycoon(commands.Cog): # Couldn't DM the degradation await ctx.send("ERROR\nYou blocked me, didn't you?") channel = ctx.channel - product = "pruner" - product_category = "tool" if not gardener.current: message = "You're currently not growing a plant." await _send_message(channel, message) else: + product = "pruner" + product_category = "tool" await self._add_health(channel, gardener, product, product_category) # async def check_degradation(self): diff --git a/reactrestrict/reactrestrict.py b/reactrestrict/reactrestrict.py index 79c3c1c..8329c46 100644 --- a/reactrestrict/reactrestrict.py +++ b/reactrestrict/reactrestrict.py @@ -98,9 +98,12 @@ class ReactRestrict(Cog): current_combos = await self.combo_list() to_keep = [ - c for c in current_combos if not (c.message_id == message_id and c.role_id == role.id) + c + for c in current_combos + if c.message_id != message_id or c.role_id != role.id ] + if to_keep != current_combos: await self.set_combo_list(to_keep) @@ -210,8 +213,7 @@ class ReactRestrict(Cog): """ Base command for this cog. Check help for the commands list. """ - if ctx.invoked_subcommand is None: - pass + pass @reactrestrict.command() async def add(self, ctx: commands.Context, message_id: int, *, role: discord.Role): diff --git a/rpsls/rpsls.py b/rpsls/rpsls.py index ed2e1dc..bada2c1 100644 --- a/rpsls/rpsls.py +++ b/rpsls/rpsls.py @@ -69,13 +69,12 @@ class RPSLS(Cog): def get_emote(self, choice): if choice == "rock": - emote = ":moyai:" + return ":moyai:" elif choice == "spock": - emote = ":vulcan:" + return ":vulcan:" elif choice == "paper": - emote = ":page_facing_up:" + return ":page_facing_up:" elif choice in ["scissors", "lizard"]: - emote = ":{}:".format(choice) + return ":{}:".format(choice) else: - emote = None - return emote + return None diff --git a/stealemoji/stealemoji.py b/stealemoji/stealemoji.py index 8f32d74..be9903c 100644 --- a/stealemoji/stealemoji.py +++ b/stealemoji/stealemoji.py @@ -69,8 +69,7 @@ class StealEmoji(Cog): """ Base command for this cog. Check help for the commands list. """ - if ctx.invoked_subcommand is None: - pass + pass @checks.is_owner() @stealemoji.command(name="clearemojis") @@ -268,37 +267,36 @@ class StealEmoji(Cog): break if guildbank is None: - if await self.config.autobank(): - try: - guildbank: discord.Guild = await self.bot.create_guild( - "StealEmoji Guildbank", code="S93bqTqKQ9rM" - ) - except discord.HTTPException: - await self.config.autobank.set(False) - log.exception("Unable to create guilds, disabling autobank") - return - async with self.config.guildbanks() as guildbanks: - guildbanks.append(guildbank.id) - # Track generated guilds for easier deletion - async with self.config.autobanked_guilds() as autobanked_guilds: - autobanked_guilds.append(guildbank.id) - - await asyncio.sleep(2) - - if guildbank.text_channels: - channel = guildbank.text_channels[0] - else: - # Always hits the else. - # Maybe create_guild doesn't return guild object with - # the template channel? - channel = await guildbank.create_text_channel("invite-channel") - invite = await channel.create_invite() - - await self.bot.send_to_owners(invite) - log.info(f"Guild created id {guildbank.id}. Invite: {invite}") - else: + if not await self.config.autobank(): + return + + try: + guildbank: discord.Guild = await self.bot.create_guild( + "StealEmoji Guildbank", code="S93bqTqKQ9rM" + ) + except discord.HTTPException: + await self.config.autobank.set(False) + log.exception("Unable to create guilds, disabling autobank") return + async with self.config.guildbanks() as guildbanks: + guildbanks.append(guildbank.id) + # Track generated guilds for easier deletion + async with self.config.autobanked_guilds() as autobanked_guilds: + autobanked_guilds.append(guildbank.id) + await asyncio.sleep(2) + + if guildbank.text_channels: + channel = guildbank.text_channels[0] + else: + # Always hits the else. + # Maybe create_guild doesn't return guild object with + # the template channel? + channel = await guildbank.create_text_channel("invite-channel") + invite = await channel.create_invite() + + await self.bot.send_to_owners(invite) + log.info(f"Guild created id {guildbank.id}. Invite: {invite}") # Next, have I saved this emoji before (because uploaded emoji != orignal emoji) if str(emoji.id) in await self.config.stolemoji(): diff --git a/timerole/timerole.py b/timerole/timerole.py index 714bcc8..b3fe843 100644 --- a/timerole/timerole.py +++ b/timerole/timerole.py @@ -77,8 +77,7 @@ class Timerole(Cog): @commands.guild_only() async def timerole(self, ctx): """Adjust timerole settings""" - if ctx.invoked_subcommand is None: - pass + pass @timerole.command() async def addrole( @@ -201,7 +200,7 @@ class Timerole(Cog): reapply = all_guilds[guild_id]["reapply"] role_dict = all_guilds[guild_id]["roles"] - if not any(role_data for role_data in role_dict.values()): # No roles + if not any(role_dict.values()): # No roles log.debug(f"No roles are configured for guild: {guild}") continue @@ -232,7 +231,7 @@ class Timerole(Cog): log.debug(f"{member.display_name} - Not time to check again yet") continue member: discord.Member - has_roles = set(r.id for r in member.roles) + has_roles = {r.id for r in member.roles} # Stop if they currently have or don't have the role, and mark had_role if (int(role_id) in has_roles and not role_data["remove"]) or ( diff --git a/unicode/unicode.py b/unicode/unicode.py index 4705f5d..297546b 100644 --- a/unicode/unicode.py +++ b/unicode/unicode.py @@ -19,8 +19,7 @@ class Unicode(Cog): @commands.group(name="unicode", pass_context=True) async def unicode(self, ctx): """Encode/Decode a Unicode character.""" - if ctx.invoked_subcommand is None: - pass + pass @unicode.command() async def decode(self, ctx: commands.Context, character): diff --git a/werewolf/builder.py b/werewolf/builder.py index da85b40..28098be 100644 --- a/werewolf/builder.py +++ b/werewolf/builder.py @@ -90,7 +90,7 @@ async def parse_code(code, game): if len(built) < digits: built += c - if built == "T" or built == "W" or built == "N": + if built in ["T", "W", "N"]: # Random Towns category = built built = "" @@ -116,8 +116,6 @@ async def parse_code(code, game): options = [role for role in ROLE_LIST if 10 + idx in role.category] elif category == "N": options = [role for role in ROLE_LIST if 20 + idx in role.category] - pass - if not options: raise IndexError("No Match Found") @@ -130,11 +128,8 @@ async def parse_code(code, game): async def encode(role_list, rand_roles): """Convert role list to code""" - out_code = "" - digit_sort = sorted(role for role in role_list if role < 10) - for role in digit_sort: - out_code += str(role) + out_code = "".join(str(role) for role in digit_sort) digit_sort = sorted(role for role in role_list if 10 <= role < 100) if digit_sort: diff --git a/werewolf/game.py b/werewolf/game.py index 668bf16..949381c 100644 --- a/werewolf/game.py +++ b/werewolf/game.py @@ -526,9 +526,10 @@ class Game: async def _notify(self, event_name, **kwargs): for i in range(1, 7): # action guide 1-6 (0 is no action) - tasks = [] - for event in self.listeners.get(event_name, {}).get(i, []): - tasks.append(asyncio.create_task(event(**kwargs))) + tasks = [ + asyncio.create_task(event(**kwargs)) + for event in self.listeners.get(event_name, {}).get(i, []) + ] # Run same-priority task simultaneously await asyncio.gather(*tasks) @@ -555,10 +556,7 @@ class Game: async def generate_targets(self, channel, with_roles=False): embed = discord.Embed(title="Remaining Players", description="[ID] - [Name]") for i, player in enumerate(self.players): - if player.alive: - status = "" - else: - status = "*[Dead]*-" + status = "" if player.alive else "*[Dead]*-" if with_roles or not player.alive: embed.add_field( name=f"{i} - {status}{player.member.display_name}", @@ -579,7 +577,7 @@ class Game: if channel_id not in self.p_channels: self.p_channels[channel_id] = self.default_secret_channel.copy() - for x in range(10): # Retry 10 times + for _ in range(10): # Retry 10 times try: await asyncio.sleep(1) # This will have multiple calls self.p_channels[channel_id]["players"].append(role.player) @@ -706,9 +704,7 @@ class Game: if not self.any_votes_remaining: await channel.send("Voting is not allowed right now") return - elif channel.name in self.p_channels: - pass - else: + elif channel.name not in self.p_channels: # Not part of the game await channel.send("Cannot vote in this channel") return @@ -757,14 +753,14 @@ class Game: await self._at_voted(target) async def eval_results(self, target, source=None, method=None): - if method is not None: - out = "**{ID}** - " + method - return out.format(ID=target.id, target=target.member.display_name) - else: + if method is None: return "**{ID}** - {target} the {role} was found dead".format( ID=target.id, target=target.member.display_name, role=await target.role.get_role() ) + out = "**{ID}** - " + method + return out.format(ID=target.id, target=target.member.display_name) + async def _quit(self, player): """ Have player quit the game diff --git a/werewolf/werewolf.py b/werewolf/werewolf.py index a4083a9..903bb54 100644 --- a/werewolf/werewolf.py +++ b/werewolf/werewolf.py @@ -75,8 +75,7 @@ class Werewolf(Cog): """ Base command to adjust settings. Check help for command list. """ - if ctx.invoked_subcommand is None: - pass + pass @commands.guild_only() @wwset.command(name="list") @@ -166,8 +165,7 @@ class Werewolf(Cog): """ Base command for this cog. Check help for the commands list. """ - if ctx.invoked_subcommand is None: - pass + pass @commands.guild_only() @ww.command(name="new") @@ -348,8 +346,7 @@ class Werewolf(Cog): """ Find custom roles by name, alignment, category, or ID """ - if ctx.invoked_subcommand is None or ctx.invoked_subcommand == self.ww_search: - pass + pass @ww_search.command(name="name") async def ww_search_name(self, ctx: commands.Context, *, name): From ea88addc423eaf52cee261ed79cdad261d00a0be Mon Sep 17 00:00:00 2001 From: bobloy Date: Thu, 18 Mar 2021 14:18:38 -0400 Subject: [PATCH 090/133] black refactoring --- ccrole/ccrole.py | 7 ++----- infochannel/infochannel.py | 5 +---- planttycoon/planttycoon.py | 9 ++------- reactrestrict/reactrestrict.py | 7 +------ 4 files changed, 6 insertions(+), 22 deletions(-) diff --git a/ccrole/ccrole.py b/ccrole/ccrole.py index e3691f8..5d1e40b 100644 --- a/ccrole/ccrole.py +++ b/ccrole/ccrole.py @@ -227,8 +227,7 @@ class CCRole(commands.Cog): if not role_list: return "None" return ", ".join( - discord.utils.get(ctx.guild.roles, id=roleid).name - for roleid in role_list + discord.utils.get(ctx.guild.roles, id=roleid).name for roleid in role_list ) embed.add_field(name="Text", value="```{}```".format(cmd["text"]), inline=False) @@ -325,9 +324,7 @@ class CCRole(commands.Cog): async def eval_cc(self, cmd, message: discord.Message, ctx: commands.Context): """Does all the work""" - if cmd["proles"] and not {role.id for role in message.author.roles} & set( - cmd["proles"] - ): + if cmd["proles"] and not {role.id for role in message.author.roles} & set(cmd["proles"]): log.debug(f"{message.author} missing required role to execute {ctx.invoked_with}") return # Not authorized, do nothing diff --git a/infochannel/infochannel.py b/infochannel/infochannel.py index fe8589f..c196e20 100644 --- a/infochannel/infochannel.py +++ b/infochannel/infochannel.py @@ -67,10 +67,7 @@ class InfoChannel(Cog): default_channel_ids = {k: None for k in self.default_channel_names} # Only members is enabled by default - default_enabled_counts = { - k: k == "members" for k in self.default_channel_names - } - + default_enabled_counts = {k: k == "members" for k in self.default_channel_names} default_guild = { "category_id": None, diff --git a/planttycoon/planttycoon.py b/planttycoon/planttycoon.py index 54d0119..0dbded9 100644 --- a/planttycoon/planttycoon.py +++ b/planttycoon/planttycoon.py @@ -249,7 +249,6 @@ class PlantTycoon(commands.Cog): if gardener.products[product] > 0 ) - degradation = ( 100 / (gardener.current["time"] / 60) @@ -306,9 +305,7 @@ class PlantTycoon(commands.Cog): damage_msg = "You used {} too many times!".format(product) else: damage_msg = "You gave too much of {}.".format(product) - message = "{} Your plant lost some health. :wilted_rose:".format( - damage_msg - ) + message = "{} Your plant lost some health. :wilted_rose:".format(damage_msg) gardener.points += self.defaults["points"]["add_health"] await gardener.save_gardener() elif product in gardener.products or product_category != "tool": @@ -440,9 +437,7 @@ class PlantTycoon(commands.Cog): if not gardener.badges: em.add_field(name="**Badges**", value="None") else: - badges = "".join( - "{}\n".format(badge.capitalize()) for badge in gardener.badges - ) + badges = "".join("{}\n".format(badge.capitalize()) for badge in gardener.badges) em.add_field(name="**Badges**", value=badges) if gardener.products: diff --git a/reactrestrict/reactrestrict.py b/reactrestrict/reactrestrict.py index 8329c46..887d4ab 100644 --- a/reactrestrict/reactrestrict.py +++ b/reactrestrict/reactrestrict.py @@ -97,12 +97,7 @@ class ReactRestrict(Cog): """ current_combos = await self.combo_list() - to_keep = [ - c - for c in current_combos - if c.message_id != message_id or c.role_id != role.id - ] - + to_keep = [c for c in current_combos if c.message_id != message_id or c.role_id != role.id] if to_keep != current_combos: await self.set_combo_list(to_keep) From dad14fe972fa9382b58adcc226d65e0cca4ad620 Mon Sep 17 00:00:00 2001 From: bobloy Date: Thu, 18 Mar 2021 16:08:10 -0400 Subject: [PATCH 091/133] black reformatting --- chatter/trainers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chatter/trainers.py b/chatter/trainers.py index 0b765b7..dc0e0b1 100644 --- a/chatter/trainers.py +++ b/chatter/trainers.py @@ -64,7 +64,7 @@ class UbuntuCorpusTrainer2(KaggleTrainer): datapath, downloadpath="ubuntu_data_v2", kaggle_dataset="rtatman/ubuntu-dialogue-corpus", - **kwargs + **kwargs, ) async def asynctrain(self, *args, **kwargs): From 8200cd9af1dfd33edd04e85e1f34af5988214501 Mon Sep 17 00:00:00 2001 From: bobloy Date: Fri, 19 Mar 2021 15:54:19 -0400 Subject: [PATCH 092/133] Run futures correctly --- chatter/chat.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/chatter/chat.py b/chatter/chat.py index 727efc2..7d3c40f 100644 --- a/chatter/chat.py +++ b/chatter/chat.py @@ -613,15 +613,13 @@ class Chatter(Cog): if in_response_to is not None: log.debug("learning response") - learning_task = asyncio.create_task( - self.loop.run_in_executor( - None, - partial( - self.chatbot.learn_response, - Statement(text), - previous_statement=in_response_to, - ), - ) + await self.loop.run_in_executor( + None, + partial( + self.chatbot.learn_response, + Statement(text), + previous_statement=in_response_to, + ), ) replying = None @@ -637,4 +635,6 @@ class Chatter(Cog): await ctx.send(":thinking:") async def check_for_kaggle(self): + """Check whether Kaggle is installed and configured properly""" + # TODO: This return False From eac7aee82c4ab29a40f79d2f1dbb16556d58672f Mon Sep 17 00:00:00 2001 From: bobloy Date: Fri, 19 Mar 2021 15:54:35 -0400 Subject: [PATCH 093/133] Save every 50 instead of all at once, so it can be cancelled --- chatter/trainers.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/chatter/trainers.py b/chatter/trainers.py index dc0e0b1..1fe5f62 100644 --- a/chatter/trainers.py +++ b/chatter/trainers.py @@ -107,19 +107,27 @@ class UbuntuCorpusTrainer2(KaggleTrainer): previous_statement_search_text = "" statements_from_file = [] + save_every = 50 + count = 0 + async for row in AsyncIter(reader): dialogue_id = row["dialogueID"] if dialogue_id != last_dialogue_id: previous_statement_text = None previous_statement_search_text = "" last_dialogue_id = dialogue_id + count += 1 + if count >= save_every: + if statements_from_file: + self.chatbot.storage.create_many(statements_from_file) + count = 0 if len(row) > 0: statement = Statement( text=row["text"], in_response_to=previous_statement_text, conversation="training", - created_at=date_parser.parse(row["date"]), + # created_at=date_parser.parse(row["date"]), persona=row["from"], ) From 04ccb435f8512b79a1c02759cd8a459d04f120a0 Mon Sep 17 00:00:00 2001 From: bobloy Date: Thu, 25 Mar 2021 09:51:41 -0400 Subject: [PATCH 094/133] Implement `check_same_thread` = False storage adapter. Add start of AsyncSQLStorageAdapter --- chatter/storage_adapters.py | 73 +++++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 chatter/storage_adapters.py diff --git a/chatter/storage_adapters.py b/chatter/storage_adapters.py new file mode 100644 index 0000000..4de2f00 --- /dev/null +++ b/chatter/storage_adapters.py @@ -0,0 +1,73 @@ +from chatterbot.storage import StorageAdapter, SQLStorageAdapter + + +class MyDumbSQLStorageAdapter(SQLStorageAdapter): + def __init__(self, **kwargs): + super(SQLStorageAdapter, self).__init__(**kwargs) + + from sqlalchemy import create_engine + from sqlalchemy.orm import sessionmaker + + self.database_uri = kwargs.get("database_uri", False) + + # None results in a sqlite in-memory database as the default + if self.database_uri is None: + self.database_uri = "sqlite://" + + # Create a file database if the database is not a connection string + if not self.database_uri: + self.database_uri = "sqlite:///db.sqlite3" + + self.engine = create_engine( + self.database_uri, convert_unicode=True, connect_args={"check_same_thread": False} + ) + + if self.database_uri.startswith("sqlite://"): + from sqlalchemy.engine import Engine + from sqlalchemy import event + + @event.listens_for(Engine, "connect") + def set_sqlite_pragma(dbapi_connection, connection_record): + dbapi_connection.execute("PRAGMA journal_mode=WAL") + dbapi_connection.execute("PRAGMA synchronous=NORMAL") + + if not self.engine.dialect.has_table(self.engine, "Statement"): + self.create_database() + + self.Session = sessionmaker(bind=self.engine, expire_on_commit=True) + + +class AsyncSQLStorageAdapter(SQLStorageAdapter): + def __init__(self, **kwargs): + super(SQLStorageAdapter, self).__init__(**kwargs) + + self.database_uri = kwargs.get("database_uri", False) + + # None results in a sqlite in-memory database as the default + if self.database_uri is None: + self.database_uri = "sqlite://" + + # Create a file database if the database is not a connection string + if not self.database_uri: + self.database_uri = "sqlite:///db.sqlite3" + + async def initialize(self): + # from sqlalchemy import create_engine + from aiomysql.sa import create_engine + from sqlalchemy.orm import sessionmaker + + self.engine = await create_engine(self.database_uri, convert_unicode=True) + + if self.database_uri.startswith("sqlite://"): + from sqlalchemy.engine import Engine + from sqlalchemy import event + + @event.listens_for(Engine, "connect") + def set_sqlite_pragma(dbapi_connection, connection_record): + dbapi_connection.execute("PRAGMA journal_mode=WAL") + dbapi_connection.execute("PRAGMA synchronous=NORMAL") + + if not self.engine.dialect.has_table(self.engine, "Statement"): + self.create_database() + + self.Session = sessionmaker(bind=self.engine, expire_on_commit=True) From 8feb21e34b70f26acf12c7d5af46e673032c9dc6 Mon Sep 17 00:00:00 2001 From: bobloy Date: Thu, 25 Mar 2021 09:52:20 -0400 Subject: [PATCH 095/133] Add new kaggle trainers --- chatter/trainers.py | 155 ++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 151 insertions(+), 4 deletions(-) diff --git a/chatter/trainers.py b/chatter/trainers.py index 1fe5f62..d8de22c 100644 --- a/chatter/trainers.py +++ b/chatter/trainers.py @@ -1,5 +1,6 @@ import asyncio import csv +import html import logging import os import pathlib @@ -56,13 +57,159 @@ class KaggleTrainer(Trainer): ), ) + def train(self, *args, **kwargs): + log.error("See asynctrain instead") -class UbuntuCorpusTrainer2(KaggleTrainer): + def asynctrain(self, *args, **kwargs): + raise self.TrainerInitializationException() + + +class SouthParkTrainer(KaggleTrainer): def __init__(self, chatbot, datapath: pathlib.Path, **kwargs): super().__init__( chatbot, datapath, downloadpath="ubuntu_data_v2", + kaggle_dataset="tovarischsukhov/southparklines", + **kwargs, + ) + + +class MovieTrainer(KaggleTrainer): + def __init__(self, chatbot, datapath: pathlib.Path, **kwargs): + super().__init__( + chatbot, + datapath, + downloadpath="kaggle_movies", + kaggle_dataset="Cornell-University/movie-dialog-corpus", + **kwargs, + ) + + async def run_movie_training(self): + dialogue_file = "movie_lines.tsv" + conversation_file = "movie_conversations.tsv" + log.info(f"Beginning dialogue training on {dialogue_file}") + start_time = time.time() + + tagger = PosLemmaTagger(language=self.chatbot.storage.tagger.language) + + # [lineID, characterID, movieID, character name, text of utterance] + # File parsing from https://www.kaggle.com/mushaya/conversation-chatbot + + with open(self.data_directory / conversation_file, "r", encoding="utf-8-sig") as conv_tsv: + conv_lines = conv_tsv.readlines() + with open(self.data_directory / dialogue_file, "r", encoding="utf-8-sig") as lines_tsv: + dialog_lines = lines_tsv.readlines() + + # trans_dict = str.maketrans({"": "__", "": "__", '""': '"'}) + + lines_dict = {} + for line in dialog_lines: + _line = line[:-1].strip('"').split("\t") + if len(_line) >= 5: # Only good lines + lines_dict[_line[0]] = ( + html.unescape(("".join(_line[4:])).strip()) + .replace("", "__") + .replace("", "__") + .replace('""', '"') + ) + else: + log.debug(f"Bad line {_line}") + + # collecting line ids for each conversation + conv = [] + for line in conv_lines[:-1]: + _line = line[:-1].split("\t")[-1][1:-1].replace("'", "").replace(" ", ",") + conv.append(_line.split(",")) + + # conversations = csv.reader(conv_tsv, delimiter="\t") + # + # reader = csv.reader(lines_tsv, delimiter="\t") + # + # + # + # lines_dict = {} + # for row in reader: + # try: + # lines_dict[row[0].strip('"')] = row[4] + # except: + # log.exception(f"Bad line: {row}") + # pass + # else: + # # print(f"Good line: {row}") + # pass + # + # # lines_dict = {row[0].strip('"'): row[4] for row in reader_list} + + statements_from_file = [] + + # [characterID of first, characterID of second, movieID, list of utterances] + async for lines in AsyncIter(conv): + previous_statement_text = None + previous_statement_search_text = "" + + for line in lines: + text = lines_dict[line] + statement = Statement( + text=text, + in_response_to=previous_statement_text, + conversation="training", + ) + + for preprocessor in self.chatbot.preprocessors: + statement = preprocessor(statement) + + statement.search_text = tagger.get_text_index_string(statement.text) + statement.search_in_response_to = previous_statement_search_text + + previous_statement_text = statement.text + previous_statement_search_text = statement.search_text + + statements_from_file.append(statement) + + if statements_from_file: + print(statements_from_file) + self.chatbot.storage.create_many(statements_from_file) + statements_from_file = [] + + print("Training took", time.time() - start_time, "seconds.") + + async def asynctrain(self, *args, **kwargs): + extracted_lines = self.data_directory / "movie_lines.tsv" + extracted_lines: pathlib.Path + + # Download and extract the Ubuntu dialog corpus if needed + if not extracted_lines.exists(): + await self.download(self.kaggle_dataset) + else: + log.info("Movie dialog already downloaded") + if not extracted_lines.exists(): + raise FileNotFoundError(f"{extracted_lines}") + + await self.run_movie_training() + + return True + + # train_dialogue = kwargs.get("train_dialogue", True) + # train_196_dialogue = kwargs.get("train_196", False) + # train_301_dialogue = kwargs.get("train_301", False) + # + # if train_dialogue: + # await self.run_dialogue_training(extracted_dir, "dialogueText.csv") + # + # if train_196_dialogue: + # await self.run_dialogue_training(extracted_dir, "dialogueText_196.csv") + # + # if train_301_dialogue: + # await self.run_dialogue_training(extracted_dir, "dialogueText_301.csv") + + +class UbuntuCorpusTrainer2(KaggleTrainer): + def __init__(self, chatbot, datapath: pathlib.Path, **kwargs): + super().__init__( + chatbot, + datapath, + downloadpath="kaggle_ubuntu", kaggle_dataset="rtatman/ubuntu-dialogue-corpus", **kwargs, ) @@ -91,6 +238,8 @@ class UbuntuCorpusTrainer2(KaggleTrainer): if train_301_dialogue: await self.run_dialogue_training(extracted_dir, "dialogueText_301.csv") + return True + async def run_dialogue_training(self, extracted_dir, dialogue_file): log.info(f"Beginning dialogue training on {dialogue_file}") start_time = time.time() @@ -120,6 +269,7 @@ class UbuntuCorpusTrainer2(KaggleTrainer): if count >= save_every: if statements_from_file: self.chatbot.storage.create_many(statements_from_file) + statements_from_file = [] count = 0 if len(row) > 0: @@ -147,9 +297,6 @@ class UbuntuCorpusTrainer2(KaggleTrainer): print("Training took", time.time() - start_time, "seconds.") - def train(self, *args, **kwargs): - log.error("See asynctrain instead") - class TwitterCorpusTrainer(Trainer): pass From ac9cf1e589308e3489a4e4b2d3759faa129009f9 Mon Sep 17 00:00:00 2001 From: bobloy Date: Thu, 25 Mar 2021 09:52:43 -0400 Subject: [PATCH 096/133] Implement movie trainer, guild cache, and learning toggle --- chatter/chat.py | 145 ++++++++++++++++++++++++++++++++++++------------ 1 file changed, 109 insertions(+), 36 deletions(-) diff --git a/chatter/chat.py b/chatter/chat.py index 7d3c40f..65966fa 100644 --- a/chatter/chat.py +++ b/chatter/chat.py @@ -17,7 +17,7 @@ from redbot.core.commands import Cog from redbot.core.data_manager import cog_data_path from redbot.core.utils.predicates import MessagePredicate -from chatter.trainers import TwitterCorpusTrainer, UbuntuCorpusTrainer2 +from chatter.trainers import MovieTrainer, TwitterCorpusTrainer, UbuntuCorpusTrainer2 log = logging.getLogger("red.fox_v3.chatter") @@ -63,6 +63,7 @@ class Chatter(Cog): "convo_delta": 15, "chatchannel": None, "reply": True, + "learning": True, } path: pathlib.Path = cog_data_path(self) self.data_path = path / "database.sqlite3" @@ -95,7 +96,8 @@ class Chatter(Cog): return ChatBot( "ChatterBot", - storage_adapter="chatterbot.storage.SQLStorageAdapter", + # storage_adapter="chatterbot.storage.SQLStorageAdapter", + storage_adapter="chatter.storage_adapters.MyDumbSQLStorageAdapter", database_uri="sqlite:///" + str(self.data_path), statement_comparison_function=self.similarity_algo, response_selection_method=get_random_response, @@ -176,10 +178,30 @@ class Chatter(Cog): trainer.train() return True - async def _train_ubuntu2(self): - trainer = UbuntuCorpusTrainer2(self.chatbot, cog_data_path(self)) + async def _train_movies(self): + trainer = MovieTrainer(self.chatbot, cog_data_path(self)) await trainer.asynctrain() + async def _train_ubuntu2(self, intensity): + train_kwarg = {} + if intensity == 196: + train_kwarg["train_dialogue"] = False + train_kwarg["train_196"] = True + elif intensity == 301: + train_kwarg["train_dialogue"] = False + train_kwarg["train_301"] = True + elif intensity == 497: + train_kwarg["train_dialogue"] = False + train_kwarg["train_196"] = True + train_kwarg["train_301"] = True + elif intensity >= 9000: # NOT 9000! + train_kwarg["train_dialogue"] = True + train_kwarg["train_196"] = True + train_kwarg["train_301"] = True + + trainer = UbuntuCorpusTrainer2(self.chatbot, cog_data_path(self)) + return await trainer.asynctrain(**train_kwarg) + def _train_english(self): trainer = ChatterBotCorpusTrainer(self.chatbot) # try: @@ -205,7 +227,7 @@ class Chatter(Cog): """ Base command for this cog. Check help for the commands list. """ - pass + self._guild_cache[ctx.guild.id] = {} # Clear cache when modifying values @commands.admin() @chatter.command(name="channel") @@ -240,19 +262,39 @@ class Chatter(Cog): await self.config.guild(ctx.guild).reply.set(toggle) if toggle: - await ctx.send("I will now respond to you if conversation continuity is not present") + await ctx.maybe_send_embed("I will now respond to you if conversation continuity is not present") else: - await ctx.send( + await ctx.maybe_send_embed( "I will not reply to your message if conversation continuity is not present, anymore" ) + @commands.admin() + @chatter.command(name="learning") + async def chatter_learning(self, ctx: commands.Context, toggle: Optional[bool] = None): + """ + Toggle the bot learning from its conversations. + + This is on by default. + """ + learning = await self.config.guild(ctx.guild).learning() + if toggle is None: + toggle = not learning + await self.config.guild(ctx.guild).learning.set(toggle) + + if toggle: + await ctx.maybe_send_embed("I will now learn from conversations.") + else: + await ctx.maybe_send_embed("I will no longer learn from conversations.") + @commands.is_owner() @chatter.command(name="cleardata") async def chatter_cleardata(self, ctx: commands.Context, confirm: bool = False): """ - This command will erase all training data and reset your configuration settings + This command will erase all training data and reset your configuration settings. - Use `[p]chatter cleardata True` + This applies to all guilds. + + Use `[p]chatter cleardata True` to confirm. """ if not confirm: @@ -364,7 +406,6 @@ class Chatter(Cog): return await self.config.guild(ctx.guild).convo_delta.set(minutes) - self._guild_cache[ctx.guild.id]["convo_delta"] = minutes await ctx.tick() @@ -420,51 +461,85 @@ class Chatter(Cog): """Commands for training the bot""" pass - @commands.is_owner() - @chatter_train.command(name="ubuntu") - async def chatter_train_ubuntu(self, ctx: commands.Context, confirmation: bool = False): + @chatter_train.group(name="kaggle") + async def chatter_train_kaggle(self, ctx: commands.Context): """ - WARNING: Large Download! Trains the bot using Ubuntu Dialog Corpus data. + Base command for kaggle training sets. + + See `[p]chatter kaggle` for details on how to enable this option + """ + pass + + @chatter_train_kaggle.command(name="ubuntu") + async def chatter_train_kaggle_ubuntu( + self, ctx: commands.Context, confirmation: bool = False, intensity=0 + ): + """ + WARNING: Large Download! Trains the bot using *NEW* Ubuntu Dialog Corpus data. """ if not confirmation: await ctx.maybe_send_embed( - "Warning: This command downloads ~500MB then eats your CPU for training\n" - "If you're sure you want to continue, run `[p]chatter train ubuntu True`" + "Warning: This command downloads ~800 then eats your CPU for training\n" + "If you're sure you want to continue, run `[p]chatter train kaggle ubuntu True`" ) return async with ctx.typing(): - future = await self.loop.run_in_executor(None, self._train_ubuntu) + future = await self._train_ubuntu2(intensity) if future: - await ctx.send("Training successful!") + await ctx.maybe_send_embed("Training successful!") else: - await ctx.send("Error occurred :(") + await ctx.maybe_send_embed("Error occurred :(") - @commands.is_owner() - @chatter_train.command(name="ubuntu2") - async def chatter_train_ubuntu2(self, ctx: commands.Context, confirmation: bool = False): + @chatter_train_kaggle.command(name="movies") + async def chatter_train_kaggle_movies(self, ctx: commands.Context, confirmation: bool = False): """ - WARNING: Large Download! Trains the bot using *NEW* Ubuntu Dialog Corpus data. + WARNING: Language! Trains the bot using Cornell University's "Movie Dialog Corpus". + + This training set contains dialog from a spread of movies with different MPAA. + This dialog includes racism, sexism, and any number of sensitive topics. + + Use at your own risk. """ if not confirmation: await ctx.maybe_send_embed( "Warning: This command downloads ~800 then eats your CPU for training\n" - "If you're sure you want to continue, run `[p]chatter train ubuntu2 True`" + "If you're sure you want to continue, run `[p]chatter train kaggle movies True`" ) return async with ctx.typing(): - future = await self._train_ubuntu2() + future = await self._train_movies() if future: - await ctx.send("Training successful!") + await ctx.maybe_send_embed("Training successful!") else: - await ctx.send("Error occurred :(") + await ctx.maybe_send_embed("Error occurred :(") + + @chatter_train.command(name="ubuntu") + async def chatter_train_ubuntu(self, ctx: commands.Context, confirmation: bool = False): + """ + WARNING: Large Download! Trains the bot using Ubuntu Dialog Corpus data. + """ + + if not confirmation: + await ctx.maybe_send_embed( + "Warning: This command downloads ~500MB then eats your CPU for training\n" + "If you're sure you want to continue, run `[p]chatter train ubuntu True`" + ) + return + + async with ctx.typing(): + future = await self.loop.run_in_executor(None, self._train_ubuntu) + + if future: + await ctx.maybe_send_embed("Training successful!") + else: + await ctx.maybe_send_embed("Error occurred :(") - @commands.is_owner() @chatter_train.command(name="english") async def chatter_train_english(self, ctx: commands.Context): """ @@ -478,7 +553,6 @@ class Chatter(Cog): else: await ctx.maybe_send_embed("Error occurred :(") - @commands.is_owner() @chatter_train.command(name="list") async def chatter_train_list(self, ctx: commands.Context): """Trains the bot based on an uploaded list. @@ -495,7 +569,6 @@ class Chatter(Cog): await ctx.send("Not yet implemented") - @commands.is_owner() @chatter_train.command(name="channel") async def chatter_train_channel(self, ctx: commands.Context, channel: discord.TextChannel): """ @@ -563,6 +636,9 @@ class Chatter(Cog): # Thank you Cog-Creators channel: discord.TextChannel = message.channel + if not self._guild_cache[guild.id]: + self._guild_cache[guild.id] = await self.config.guild(guild).all() + is_reply = False # this is only useful with in_response_to if ( message.reference is not None @@ -571,7 +647,7 @@ class Chatter(Cog): ): is_reply = True # this is only useful with in_response_to pass # this is a reply to the bot, good to go - elif guild is not None and channel.id == await self.config.guild(guild).chatchannel(): + elif guild is not None and channel.id == self._guild_cache[guild.id]["chatchannel"]: pass # good to go else: when_mentionables = commands.when_mentioned(self.bot, message) @@ -588,9 +664,6 @@ class Chatter(Cog): async with ctx.typing(): - if not self._guild_cache[ctx.guild.id]: - self._guild_cache[ctx.guild.id] = await self.config.guild(ctx.guild).all() - if is_reply: in_response_to = message.reference.resolved.content elif self._last_message_per_channel[ctx.channel.id] is not None: @@ -611,7 +684,7 @@ class Chatter(Cog): None, self.chatbot.generate_response, Statement(text) ) - if in_response_to is not None: + if in_response_to is not None and self._guild_cache[guild.id]["learning"]: log.debug("learning response") await self.loop.run_in_executor( None, @@ -623,7 +696,7 @@ class Chatter(Cog): ) replying = None - if await self.config.guild(guild).reply(): + if self._guild_cache[guild.id]["reply"]: if message != ctx.channel.last_message: replying = message From b4f20dd7d283ed64ab7429824839a533c8abf2e7 Mon Sep 17 00:00:00 2001 From: bobloy Date: Thu, 25 Mar 2021 09:54:14 -0400 Subject: [PATCH 097/133] Don't print everything, use log --- chatter/trainers.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/chatter/trainers.py b/chatter/trainers.py index d8de22c..962fa08 100644 --- a/chatter/trainers.py +++ b/chatter/trainers.py @@ -136,7 +136,7 @@ class MovieTrainer(KaggleTrainer): # log.exception(f"Bad line: {row}") # pass # else: - # # print(f"Good line: {row}") + # # log.info(f"Good line: {row}") # pass # # # lines_dict = {row[0].strip('"'): row[4] for row in reader_list} @@ -168,11 +168,10 @@ class MovieTrainer(KaggleTrainer): statements_from_file.append(statement) if statements_from_file: - print(statements_from_file) self.chatbot.storage.create_many(statements_from_file) statements_from_file = [] - print("Training took", time.time() - start_time, "seconds.") + log.info("Training took", time.time() - start_time, "seconds.") async def asynctrain(self, *args, **kwargs): extracted_lines = self.data_directory / "movie_lines.tsv" @@ -295,7 +294,7 @@ class UbuntuCorpusTrainer2(KaggleTrainer): if statements_from_file: self.chatbot.storage.create_many(statements_from_file) - print("Training took", time.time() - start_time, "seconds.") + log.info("Training took", time.time() - start_time, "seconds.") class TwitterCorpusTrainer(Trainer): From 59fd96fc5af9d1a0ab9e5c70199f40369381c6ba Mon Sep 17 00:00:00 2001 From: bobloy Date: Thu, 25 Mar 2021 10:01:56 -0400 Subject: [PATCH 098/133] add save_every for less disk intensive work. --- chatter/trainers.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/chatter/trainers.py b/chatter/trainers.py index 962fa08..adf042f 100644 --- a/chatter/trainers.py +++ b/chatter/trainers.py @@ -142,6 +142,8 @@ class MovieTrainer(KaggleTrainer): # # lines_dict = {row[0].strip('"'): row[4] for row in reader_list} statements_from_file = [] + save_every = 50 + count = 0 # [characterID of first, characterID of second, movieID, list of utterances] async for lines in AsyncIter(conv): @@ -167,9 +169,15 @@ class MovieTrainer(KaggleTrainer): statements_from_file.append(statement) - if statements_from_file: - self.chatbot.storage.create_many(statements_from_file) - statements_from_file = [] + count += 1 + if count >= save_every: + if statements_from_file: + self.chatbot.storage.create_many(statements_from_file) + statements_from_file = [] + count = 0 + + if statements_from_file: + self.chatbot.storage.create_many(statements_from_file) log.info("Training took", time.time() - start_time, "seconds.") From 802929d757458f9ff4fac99203ced1f033c9bdbc Mon Sep 17 00:00:00 2001 From: bobloy Date: Thu, 25 Mar 2021 10:02:02 -0400 Subject: [PATCH 099/133] better wording --- chatter/chat.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/chatter/chat.py b/chatter/chat.py index 65966fa..9e3379f 100644 --- a/chatter/chat.py +++ b/chatter/chat.py @@ -480,7 +480,7 @@ class Chatter(Cog): if not confirmation: await ctx.maybe_send_embed( - "Warning: This command downloads ~800 then eats your CPU for training\n" + "Warning: This command downloads ~800MB and is CPU intensive during training\n" "If you're sure you want to continue, run `[p]chatter train kaggle ubuntu True`" ) return @@ -506,7 +506,7 @@ class Chatter(Cog): if not confirmation: await ctx.maybe_send_embed( - "Warning: This command downloads ~800 then eats your CPU for training\n" + "Warning: This command downloads ~29MB and is CPU intensive during training\n" "If you're sure you want to continue, run `[p]chatter train kaggle movies True`" ) return @@ -527,7 +527,7 @@ class Chatter(Cog): if not confirmation: await ctx.maybe_send_embed( - "Warning: This command downloads ~500MB then eats your CPU for training\n" + "Warning: This command downloads ~500MB and is CPU intensive during training\n" "If you're sure you want to continue, run `[p]chatter train ubuntu True`" ) return From 1319d98972e0b79677a34585fc0b1be2786802e2 Mon Sep 17 00:00:00 2001 From: bobloy Date: Thu, 25 Mar 2021 10:56:48 -0400 Subject: [PATCH 100/133] Less often, still writing too much. --- chatter/trainers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chatter/trainers.py b/chatter/trainers.py index adf042f..4f80b79 100644 --- a/chatter/trainers.py +++ b/chatter/trainers.py @@ -142,7 +142,7 @@ class MovieTrainer(KaggleTrainer): # # lines_dict = {row[0].strip('"'): row[4] for row in reader_list} statements_from_file = [] - save_every = 50 + save_every = 300 count = 0 # [characterID of first, characterID of second, movieID, list of utterances] From db24bb4db4f81d2248b82219d7953798be4dc585 Mon Sep 17 00:00:00 2001 From: bobloy Date: Thu, 25 Mar 2021 10:57:35 -0400 Subject: [PATCH 101/133] No differences --- chatter/chat.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/chatter/chat.py b/chatter/chat.py index 9e3379f..fe50588 100644 --- a/chatter/chat.py +++ b/chatter/chat.py @@ -262,7 +262,9 @@ class Chatter(Cog): await self.config.guild(ctx.guild).reply.set(toggle) if toggle: - await ctx.maybe_send_embed("I will now respond to you if conversation continuity is not present") + await ctx.maybe_send_embed( + "I will now respond to you if conversation continuity is not present" + ) else: await ctx.maybe_send_embed( "I will not reply to your message if conversation continuity is not present, anymore" From 87187abbb3423fc6539864c4239b858d54b280e7 Mon Sep 17 00:00:00 2001 From: bobloy Date: Thu, 25 Mar 2021 11:11:57 -0400 Subject: [PATCH 102/133] Fix logging --- chatter/trainers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/chatter/trainers.py b/chatter/trainers.py index 4f80b79..3cc92da 100644 --- a/chatter/trainers.py +++ b/chatter/trainers.py @@ -179,7 +179,7 @@ class MovieTrainer(KaggleTrainer): if statements_from_file: self.chatbot.storage.create_many(statements_from_file) - log.info("Training took", time.time() - start_time, "seconds.") + log.info(f"Training took {time.time() - start_time} seconds.") async def asynctrain(self, *args, **kwargs): extracted_lines = self.data_directory / "movie_lines.tsv" @@ -302,7 +302,7 @@ class UbuntuCorpusTrainer2(KaggleTrainer): if statements_from_file: self.chatbot.storage.create_many(statements_from_file) - log.info("Training took", time.time() - start_time, "seconds.") + log.info(f"Training took {time.time() - start_time} seconds.") class TwitterCorpusTrainer(Trainer): From e1297a4dcaec7b12bc1958a728f47d96cfdac5dc Mon Sep 17 00:00:00 2001 From: bobloy Date: Thu, 25 Mar 2021 11:12:05 -0400 Subject: [PATCH 103/133] Return success value --- chatter/chat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chatter/chat.py b/chatter/chat.py index fe50588..d999d94 100644 --- a/chatter/chat.py +++ b/chatter/chat.py @@ -180,7 +180,7 @@ class Chatter(Cog): async def _train_movies(self): trainer = MovieTrainer(self.chatbot, cog_data_path(self)) - await trainer.asynctrain() + return await trainer.asynctrain() async def _train_ubuntu2(self, intensity): train_kwarg = {} From 9f22dfb790ab8421dd205ff0d975e8bf448cc7f8 Mon Sep 17 00:00:00 2001 From: bobloy Date: Thu, 25 Mar 2021 17:24:16 -0400 Subject: [PATCH 104/133] Swap learning to global config --- chatter/chat.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/chatter/chat.py b/chatter/chat.py index d999d94..9caf050 100644 --- a/chatter/chat.py +++ b/chatter/chat.py @@ -56,14 +56,13 @@ class Chatter(Cog): super().__init__() self.bot = bot self.config = Config.get_conf(self, identifier=6710497116116101114) - default_global = {} + default_global = {"learning": True} default_guild = { "whitelist": None, "days": 1, "convo_delta": 15, "chatchannel": None, "reply": True, - "learning": True, } path: pathlib.Path = cog_data_path(self) self.data_path = path / "database.sqlite3" @@ -85,6 +84,7 @@ class Chatter(Cog): self.loop = asyncio.get_event_loop() self._guild_cache = defaultdict(dict) + self._global_cache = {} self._last_message_per_channel: Dict[Optional[discord.Message]] = defaultdict(lambda: None) @@ -228,6 +228,7 @@ class Chatter(Cog): Base command for this cog. Check help for the commands list. """ self._guild_cache[ctx.guild.id] = {} # Clear cache when modifying values + self._global_cache = {} @commands.admin() @chatter.command(name="channel") @@ -270,18 +271,19 @@ class Chatter(Cog): "I will not reply to your message if conversation continuity is not present, anymore" ) - @commands.admin() + @commands.is_owner() @chatter.command(name="learning") async def chatter_learning(self, ctx: commands.Context, toggle: Optional[bool] = None): """ Toggle the bot learning from its conversations. + This is a global setting. This is on by default. """ - learning = await self.config.guild(ctx.guild).learning() + learning = await self.config.learning() if toggle is None: toggle = not learning - await self.config.guild(ctx.guild).learning.set(toggle) + await self.config.learning.set(toggle) if toggle: await ctx.maybe_send_embed("I will now learn from conversations.") @@ -686,7 +688,10 @@ class Chatter(Cog): None, self.chatbot.generate_response, Statement(text) ) - if in_response_to is not None and self._guild_cache[guild.id]["learning"]: + if not self._global_cache: + self._global_cache = await self.config.all() + + if in_response_to is not None and self._global_cache["learning"]: log.debug("learning response") await self.loop.run_in_executor( None, From 5f30bc1234995f371ff35ad175714ed6e618723c Mon Sep 17 00:00:00 2001 From: bobloy Date: Fri, 26 Mar 2021 13:53:00 -0400 Subject: [PATCH 105/133] Add multiple channel training --- chatter/chat.py | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/chatter/chat.py b/chatter/chat.py index 9caf050..8ecbf1a 100644 --- a/chatter/chat.py +++ b/chatter/chat.py @@ -5,7 +5,7 @@ import pathlib from collections import defaultdict from datetime import datetime, timedelta from functools import partial -from typing import Dict, Optional +from typing import Dict, List, Optional import discord from chatterbot import ChatBot @@ -107,7 +107,7 @@ class Chatter(Cog): logger=log, ) - async def _get_conversation(self, ctx, in_channel: discord.TextChannel = None): + async def _get_conversation(self, ctx, in_channels: List[discord.TextChannel]): """ Compiles all conversation in the Guild this bot can get it's hands on Currently takes a stupid long time @@ -124,9 +124,9 @@ class Chatter(Cog): # Should always be positive numbers return msg.created_at - sent >= delta - for channel in ctx.guild.text_channels: - if in_channel: - channel = in_channel + for channel in in_channels: + # if in_channel: + # channel = in_channel await ctx.maybe_send_embed("Gathering {}".format(channel.mention)) user = None i = 0 @@ -161,8 +161,8 @@ class Chatter(Cog): except discord.HTTPException: pass - if in_channel: - break + # if in_channel: + # break return out @@ -574,10 +574,15 @@ class Chatter(Cog): await ctx.send("Not yet implemented") @chatter_train.command(name="channel") - async def chatter_train_channel(self, ctx: commands.Context, channel: discord.TextChannel): + async def chatter_train_channel( + self, ctx: commands.Context, channels: commands.Greedy[discord.TextChannel] + ): """ Trains the bot based on language in this guild. """ + if not channels: + await ctx.send_help() + return await ctx.maybe_send_embed( "Warning: The cog may use significant RAM or CPU if trained on large data sets.\n" @@ -586,7 +591,7 @@ class Chatter(Cog): ) async with ctx.typing(): - conversation = await self._get_conversation(ctx, channel) + conversation = await self._get_conversation(ctx, channels) if not conversation: await ctx.maybe_send_embed("Failed to gather training data") From 93b403b35fb4c17b89b5e52218c2c39562c825eb Mon Sep 17 00:00:00 2001 From: bobloy Date: Mon, 5 Apr 2021 09:25:23 -0400 Subject: [PATCH 106/133] Better progress logging --- chatter/chat.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/chatter/chat.py b/chatter/chat.py index 8ecbf1a..5ad3efb 100644 --- a/chatter/chat.py +++ b/chatter/chat.py @@ -213,13 +213,10 @@ class Chatter(Cog): def _train(self, data): trainer = ListTrainer(self.chatbot) total = len(data) - # try: for c, convo in enumerate(data, 1): + log.info(f"{c} / {total}") if len(convo) > 1: # TODO: Toggleable skipping short conversations - print(f"{c} / {total}") trainer.train(convo) - # except: - # return False return True @commands.group(invoke_without_command=False) From 10ed1f9b9f34315fecf406aff8666676d7a68bca Mon Sep 17 00:00:00 2001 From: bobloy Date: Wed, 14 Apr 2021 09:40:34 -0400 Subject: [PATCH 107/133] pagify lots of stolen emojis --- stealemoji/stealemoji.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/stealemoji/stealemoji.py b/stealemoji/stealemoji.py index be9903c..b456616 100644 --- a/stealemoji/stealemoji.py +++ b/stealemoji/stealemoji.py @@ -6,6 +6,7 @@ import discord from redbot.core import Config, checks, commands from redbot.core.bot import Red from redbot.core.commands import Cog +from redbot.core.utils.chat_formatting import pagify log = logging.getLogger("red.fox_v3.stealemoji") # Replaced with discord.Asset.read() @@ -99,7 +100,8 @@ class StealEmoji(Cog): await ctx.maybe_send_embed("No stolen emojis yet") return - await ctx.maybe_send_embed(emoj) + for page in pagify(emoj): + await ctx.maybe_send_embed(page) @checks.is_owner() @stealemoji.command(name="notify") From 28edcc1fddae07a17d91973efb51edfc5ba95280 Mon Sep 17 00:00:00 2001 From: bobloy Date: Wed, 14 Apr 2021 09:44:28 -0400 Subject: [PATCH 108/133] deliminate on space not newline --- stealemoji/stealemoji.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stealemoji/stealemoji.py b/stealemoji/stealemoji.py index b456616..fb83f3b 100644 --- a/stealemoji/stealemoji.py +++ b/stealemoji/stealemoji.py @@ -100,7 +100,7 @@ class StealEmoji(Cog): await ctx.maybe_send_embed("No stolen emojis yet") return - for page in pagify(emoj): + for page in pagify(emoj, delims=[" "]): await ctx.maybe_send_embed(page) @checks.is_owner() From f04ff6886b2382a59ea19d187611b6ae25768abe Mon Sep 17 00:00:00 2001 From: Kreusada <67752638+Kreusada@users.noreply.github.com> Date: Thu, 15 Apr 2021 13:59:19 +0100 Subject: [PATCH 109/133] [SCP] Remove double setup --- scp/scp.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/scp/scp.py b/scp/scp.py index 3b4176c..d564d4c 100644 --- a/scp/scp.py +++ b/scp/scp.py @@ -177,7 +177,3 @@ class SCP(Cog): msg = "http://www.scp-wiki.net/log-of-unexplained-locations" await ctx.maybe_send_embed(msg) - - -def setup(bot): - bot.add_cog(SCP(bot)) From 1ddcd980789c819f6a1db5ac08423376b1b5be6f Mon Sep 17 00:00:00 2001 From: bobloy Date: Thu, 15 Apr 2021 11:12:47 -0400 Subject: [PATCH 110/133] Implement languages --- tts/tts.py | 53 +++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 49 insertions(+), 4 deletions(-) diff --git a/tts/tts.py b/tts/tts.py index 8584f5a..fe86407 100644 --- a/tts/tts.py +++ b/tts/tts.py @@ -1,11 +1,40 @@ import io +import logging +from typing import Optional, TYPE_CHECKING import discord +import pycountry +from discord.ext.commands import BadArgument, Converter from gtts import gTTS +from gtts.lang import _fallback_deprecated_lang, tts_langs from redbot.core import Config, commands from redbot.core.bot import Red from redbot.core.commands import Cog +log = logging.getLogger("red.fox_v3.tts") + +langs = [lang.iso639_1_code for lang in pycountry.languages if hasattr(lang, "iso639_1_code")] + +print(langs) + +if TYPE_CHECKING: + ISO639Converter = str +else: + + class ISO639Converter(Converter): + async def convert(self, ctx, argument) -> str: + lang = _fallback_deprecated_lang(argument) + + try: + langs = tts_langs() + if lang not in langs: + raise BadArgument("Language not supported: %s" % lang) + except RuntimeError as e: + log.debug(str(e), exc_info=True) + log.warning(str(e)) + + return lang + class TTS(Cog): """ @@ -18,7 +47,7 @@ class TTS(Cog): self.config = Config.get_conf(self, identifier=9811198108111121, force_registration=True) default_global = {} - default_guild = {} + default_guild = {"language": "en"} self.config.register_global(**default_global) self.config.register_guild(**default_guild) @@ -27,11 +56,27 @@ class TTS(Cog): """Nothing to delete""" return + @commands.mod() + @commands.command() + async def ttslang(self, ctx: commands.Context, lang: ISO639Converter): + """ + Sets the default language for TTS in this guild. + + Default is `en` for English + """ + await self.config.guild(ctx.guild).language.set(lang) + await ctx.send(f"Default tts language set to {lang}") + @commands.command(aliases=["t2s", "text2"]) - async def tts(self, ctx: commands.Context, *, lang: str, text: str): + async def tts( + self, ctx: commands.Context, lang: Optional[ISO639Converter] = None, *, text: str + ): + """ + Send Text to speech messages as an mp3 """ - Send Text to speech messages as an mp3 - """ + if lang is None: + lang = await self.config.guild(ctx.guild).language() + mp3_fp = io.BytesIO() tts = gTTS(text, lang=lang) tts.write_to_fp(mp3_fp) From 506a79c6d6294c0353173e04f6b9c7bcb7d54bae Mon Sep 17 00:00:00 2001 From: bobloy Date: Thu, 15 Apr 2021 11:14:00 -0400 Subject: [PATCH 111/133] Removing debugging print --- tts/tts.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tts/tts.py b/tts/tts.py index fe86407..02d7267 100644 --- a/tts/tts.py +++ b/tts/tts.py @@ -15,8 +15,6 @@ log = logging.getLogger("red.fox_v3.tts") langs = [lang.iso639_1_code for lang in pycountry.languages if hasattr(lang, "iso639_1_code")] -print(langs) - if TYPE_CHECKING: ISO639Converter = str else: From ed6cc433c8891cac1e0f1353088bbe10b6fcb146 Mon Sep 17 00:00:00 2001 From: bobloy Date: Thu, 15 Apr 2021 11:38:36 -0400 Subject: [PATCH 112/133] Remove pycountry --- tts/tts.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tts/tts.py b/tts/tts.py index 02d7267..c69522a 100644 --- a/tts/tts.py +++ b/tts/tts.py @@ -3,7 +3,6 @@ import logging from typing import Optional, TYPE_CHECKING import discord -import pycountry from discord.ext.commands import BadArgument, Converter from gtts import gTTS from gtts.lang import _fallback_deprecated_lang, tts_langs @@ -13,8 +12,6 @@ from redbot.core.commands import Cog log = logging.getLogger("red.fox_v3.tts") -langs = [lang.iso639_1_code for lang in pycountry.languages if hasattr(lang, "iso639_1_code")] - if TYPE_CHECKING: ISO639Converter = str else: From 2f21de6a972a6e55b9f6e85616ea462ee3774490 Mon Sep 17 00:00:00 2001 From: PhenoM4n4n Date: Thu, 15 Apr 2021 17:39:49 -0700 Subject: [PATCH 113/133] lovecalc attributeerror --- lovecalculator/lovecalculator.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lovecalculator/lovecalculator.py b/lovecalculator/lovecalculator.py index 20504bd..a706ea0 100644 --- a/lovecalculator/lovecalculator.py +++ b/lovecalculator/lovecalculator.py @@ -49,7 +49,11 @@ class LoveCalculator(Cog): result_image = soup_object.find("img", class_="result__image").get("src") - result_text = soup_object.find("div", class_="result-text").get_text() + result_text = soup_object.find("div", class_="result-text") + if result_text is None: + result_text = f"{x} and {y} aren't compatible 😔" + else: + result_text.get_text() result_text = " ".join(result_text.split()) try: From 2937b6ac923d7c16c79d907d88b143408b3dac08 Mon Sep 17 00:00:00 2001 From: bobloy Date: Mon, 3 May 2021 13:58:35 -0400 Subject: [PATCH 114/133] List of dicts --- fifo/task.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fifo/task.py b/fifo/task.py index e1b7207..34df8e2 100644 --- a/fifo/task.py +++ b/fifo/task.py @@ -142,7 +142,7 @@ class FakeMessage(discord.Message): self._update( { "mention_roles": self.raw_role_mentions, - "mentions": self.raw_mentions, + "mentions": [{"id": _id} for _id in self.raw_mentions], } ) From 52ca2f6a4541a1d4b1ee8029175dfb61781b5ebb Mon Sep 17 00:00:00 2001 From: aleclol <50505980+aleclol@users.noreply.github.com> Date: Tue, 18 May 2021 15:27:55 -0400 Subject: [PATCH 115/133] Update lovecalculator.py --- lovecalculator/lovecalculator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lovecalculator/lovecalculator.py b/lovecalculator/lovecalculator.py index a706ea0..d6ae4fe 100644 --- a/lovecalculator/lovecalculator.py +++ b/lovecalculator/lovecalculator.py @@ -53,7 +53,7 @@ class LoveCalculator(Cog): if result_text is None: result_text = f"{x} and {y} aren't compatible 😔" else: - result_text.get_text() + result_text = result_text.get_text() result_text = " ".join(result_text.split()) try: From bc12aa866e6d7f39304945217407405abc42a9fe Mon Sep 17 00:00:00 2001 From: bobloy Date: Tue, 25 May 2021 10:12:51 -0400 Subject: [PATCH 116/133] Fix formatting --- timerole/timerole.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/timerole/timerole.py b/timerole/timerole.py index b3fe843..0d62cf0 100644 --- a/timerole/timerole.py +++ b/timerole/timerole.py @@ -295,8 +295,11 @@ class Timerole(Cog): log.exception("Failed Adding Roles") add_results += f"{member.display_name} : **(Failed Adding Roles)**\n" else: - add_results += " \n".join( - f"{member.display_name} : {role.name}" for role in add_roles + add_results += ( + " \n".join( + f"{member.display_name} : {role.name}" for role in add_roles + ) + + "\n" ) for role_id in addlist: await self.config.custom( @@ -310,8 +313,11 @@ class Timerole(Cog): log.exception("Failed Removing Roles") remove_results += f"{member.display_name} : **(Failed Removing Roles)**\n" else: - remove_results += " \n".join( - f"{member.display_name} : {role.name}" for role in remove_roles + remove_results += ( + " \n".join( + f"{member.display_name} : {role.name}" for role in remove_roles + ) + + "\n" ) for role_id in removelist: await self.config.custom( From a55ae8a51126099a3418ce7f9b6889e5f233bdf9 Mon Sep 17 00:00:00 2001 From: bobloy Date: Wed, 30 Jun 2021 17:29:00 -0400 Subject: [PATCH 117/133] Use Bobloy's chatterbot fork --- chatter/README.md | 66 +++++++------------------------------ chatter/chat.py | 8 ++++- chatter/info.json | 12 +------ chatter/requirements.txt | 24 +++++++------- chatter/storage_adapters.py | 2 +- 5 files changed, 33 insertions(+), 79 deletions(-) diff --git a/chatter/README.md b/chatter/README.md index 06331b2..bcd6cbc 100644 --- a/chatter/README.md +++ b/chatter/README.md @@ -74,77 +74,35 @@ If you get an error at this step, stop and skip to one of the manual methods bel #### Step 2: Install additional dependencies -Assuming the previous commands had no error, you can now use `pipinstall` to add the remaining dependencies. +Here you need to decide which training models you want to have available to you. -NOTE: This method is not the intended use case for `pipinstall` and may stop working in the future. +Shutdown the bot and run any number of these in the console: ``` -[p]pipinstall --no-deps chatterbot>=1.1 -``` - -#### Step 3: Load the cog and get started - -``` -[p]load chatter -``` +python -m spacy download en_core_web_sm # ~15 MB -### Windows - Manually -#### Step 1: Built-in Downloader +python -m spacy download en_core_web_md # ~50 MB -You need to get a copy of the requirements.txt provided with chatter, I recommend this method. +python -m spacy download en_core_web_lg # ~750 MB (CPU Optimized) -``` -[p]repo add Fox https://github.com/bobloy/Fox-V3 +python -m spacy download en_core_web_trf # ~500 MB (GPU Optimized) ``` -#### Step 2: Install Requirements - -Make sure you have your virtual environment that you installed Red on activated before starting this step. See the Red Docs for details on how. - -In a terminal running as an admin, navigate to the directory containing this repo. - -I've used my install directory as an example. - -``` -cd C:\Users\Bobloy\AppData\Local\Red-DiscordBot\Red-DiscordBot\data\bobbot\cogs\RepoManager\repos\Fox\chatter -pip install -r requirements.txt -pip install --no-deps "chatterbot>=1.1" -``` - -#### Step 3: Load Chatter +#### Step 3: Load the cog and get started ``` -[p]repo add Fox https://github.com/bobloy/Fox-V3 # If you didn't already do this in step 1 -[p]cog install Fox chatter [p]load chatter ``` -### Linux - Manually - -#### Step 1: Built-in Downloader - -``` -[p]repo add Fox https://github.com/bobloy/Fox-V3 -[p]cog install Fox chatter -``` - -#### Step 2: Install Requirements - -In your console with your virtual environment activated: - -``` -pip install --no-deps "chatterbot>=1.1" -``` - -### Step 3: Load Chatter +### Windows - Manually +Deprecated -``` -[p]load chatter -``` +### Linux - Manually +Deprecated # Configuration -Chatter works out the the box without any training by learning as it goes, +Chatter works out the box without any training by learning as it goes, but will have very poor and repetitive responses at first. Initial training is recommended to speed up its learning. diff --git a/chatter/chat.py b/chatter/chat.py index 5ad3efb..b8cf75d 100644 --- a/chatter/chat.py +++ b/chatter/chat.py @@ -29,6 +29,12 @@ def my_local_get_prefix(prefixes, content): return None +class ENG_TRF: + ISO_639_1 = "en_core_web_trf" + ISO_639 = "eng" + ENGLISH_NAME = "English" + + class ENG_LG: ISO_639_1 = "en_core_web_lg" ISO_639 = "eng" @@ -70,7 +76,7 @@ class Chatter(Cog): # TODO: Move training_model and similarity_algo to config # TODO: Add an option to see current settings - self.tagger_language = ENG_MD + self.tagger_language = ENG_TRF self.similarity_algo = SpacySimilarity self.similarity_threshold = 0.90 self.chatbot = self._create_chatbot() diff --git a/chatter/info.json b/chatter/info.json index fc31e7c..a9bb96b 100644 --- a/chatter/info.json +++ b/chatter/info.json @@ -7,17 +7,7 @@ "hidden": false, "install_msg": "Thank you for installing Chatter! Please make sure you check the install instructions at https://github.com/bobloy/Fox-V3/blob/master/chatter/README.md\nAfter that, get started ith `[p]load chatter` and `[p]help Chatter`", "requirements": [ - "git+git://github.com/gunthercox/chatterbot-corpus@master#egg=chatterbot_corpus", - "mathparse>=0.1,<0.2", - "nltk>=3.2,<4.0", - "pint>=0.8.1", - "python-dateutil>=2.8,<2.9", - "pyyaml>=5.3,<5.4", - "sqlalchemy>=1.3,<1.4", - "pytz", - "https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-2.3.1/en_core_web_sm-2.3.1.tar.gz#egg=en_core_web_sm", - "https://github.com/explosion/spacy-models/releases/download/en_core_web_md-2.3.1/en_core_web_md-2.3.1.tar.gz#egg=en_core_web_md", - "spacy>=2.3,<2.4", + "git+git://github.com/bobloy/ChatterBot@fox#egg=ChatterBot", "kaggle" ], "short": "Local Chatbot run on machine learning", diff --git a/chatter/requirements.txt b/chatter/requirements.txt index 88cd662..bbcfcf9 100644 --- a/chatter/requirements.txt +++ b/chatter/requirements.txt @@ -1,12 +1,12 @@ -git+git://github.com/gunthercox/chatterbot-corpus@master#egg=chatterbot_corpus -mathparse>=0.1,<0.2 -nltk>=3.2,<4.0 -pint>=0.8.1 -python-dateutil>=2.8,<2.9 -pyyaml>=5.3,<5.4 -sqlalchemy>=1.3,<1.4 -pytz -spacy>=2.3,<2.4 -https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-2.3.1/en_core_web_sm-2.3.1.tar.gz#egg=en_core_web_sm -https://github.com/explosion/spacy-models/releases/download/en_core_web_md-2.3.1/en_core_web_md-2.3.1.tar.gz#egg=en_core_web_md -# https://github.com/explosion/spacy-models/releases/download/en_core_web_lg-2.3.1/en_core_web_lg-2.3.1.tar.gz#egg=en_core_web_lg \ No newline at end of file +# git+git://github.com/gunthercox/chatterbot-corpus@master#egg=chatterbot_corpus +# mathparse>=0.1,<0.2 +# nltk>=3.2,<4.0 +# pint>=0.8.1 +# python-dateutil>=2.8,<2.9 +# # pyyaml>=5.3,<5.4 +# sqlalchemy>=1.3,<1.4 +# pytz +# spacy>=2.3,<2.4 +# https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-2.3.1/en_core_web_sm-2.3.1.tar.gz#egg=en_core_web_sm +# https://github.com/explosion/spacy-models/releases/download/en_core_web_md-2.3.1/en_core_web_md-2.3.1.tar.gz#egg=en_core_web_md +# # https://github.com/explosion/spacy-models/releases/download/en_core_web_lg-2.3.1/en_core_web_lg-2.3.1.tar.gz#egg=en_core_web_lg \ No newline at end of file diff --git a/chatter/storage_adapters.py b/chatter/storage_adapters.py index 4de2f00..6f11601 100644 --- a/chatter/storage_adapters.py +++ b/chatter/storage_adapters.py @@ -19,7 +19,7 @@ class MyDumbSQLStorageAdapter(SQLStorageAdapter): self.database_uri = "sqlite:///db.sqlite3" self.engine = create_engine( - self.database_uri, convert_unicode=True, connect_args={"check_same_thread": False} + self.database_uri, connect_args={"check_same_thread": False} ) if self.database_uri.startswith("sqlite://"): From 9c63c126563153e3c5290bc5d6c2dde06c1a4c37 Mon Sep 17 00:00:00 2001 From: bobloy Date: Wed, 30 Jun 2021 17:29:31 -0400 Subject: [PATCH 118/133] Don't need requirements file --- chatter/requirements.txt | 12 ------------ 1 file changed, 12 deletions(-) delete mode 100644 chatter/requirements.txt diff --git a/chatter/requirements.txt b/chatter/requirements.txt deleted file mode 100644 index bbcfcf9..0000000 --- a/chatter/requirements.txt +++ /dev/null @@ -1,12 +0,0 @@ -# git+git://github.com/gunthercox/chatterbot-corpus@master#egg=chatterbot_corpus -# mathparse>=0.1,<0.2 -# nltk>=3.2,<4.0 -# pint>=0.8.1 -# python-dateutil>=2.8,<2.9 -# # pyyaml>=5.3,<5.4 -# sqlalchemy>=1.3,<1.4 -# pytz -# spacy>=2.3,<2.4 -# https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-2.3.1/en_core_web_sm-2.3.1.tar.gz#egg=en_core_web_sm -# https://github.com/explosion/spacy-models/releases/download/en_core_web_md-2.3.1/en_core_web_md-2.3.1.tar.gz#egg=en_core_web_md -# # https://github.com/explosion/spacy-models/releases/download/en_core_web_lg-2.3.1/en_core_web_lg-2.3.1.tar.gz#egg=en_core_web_lg \ No newline at end of file From c165313031644c4480aedff848859a7a1c36da8b Mon Sep 17 00:00:00 2001 From: bobloy Date: Thu, 1 Jul 2021 15:25:38 -0400 Subject: [PATCH 119/133] no reply errors in cache --- chatter/chat.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/chatter/chat.py b/chatter/chat.py index b8cf75d..2deb082 100644 --- a/chatter/chat.py +++ b/chatter/chat.py @@ -63,7 +63,7 @@ class Chatter(Cog): self.bot = bot self.config = Config.get_conf(self, identifier=6710497116116101114) default_global = {"learning": True} - default_guild = { + self.default_guild = { "whitelist": None, "days": 1, "convo_delta": 15, @@ -711,7 +711,9 @@ class Chatter(Cog): ) replying = None - if self._guild_cache[guild.id]["reply"]: + if ( + "reply" not in self._guild_cache[guild.id] and self.default_guild["reply"] + ) or self._guild_cache[guild.id]["reply"]: if message != ctx.channel.last_message: replying = message From 6f0c88b1ac31b424de5af46cade5b6f1fb816879 Mon Sep 17 00:00:00 2001 From: bobloy Date: Thu, 1 Jul 2021 17:19:29 -0400 Subject: [PATCH 120/133] change default models, fix requirements --- chatter/chat.py | 9 +++++---- chatter/info.json | 1 + 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/chatter/chat.py b/chatter/chat.py index 2deb082..e3f14cc 100644 --- a/chatter/chat.py +++ b/chatter/chat.py @@ -76,7 +76,7 @@ class Chatter(Cog): # TODO: Move training_model and similarity_algo to config # TODO: Add an option to see current settings - self.tagger_language = ENG_TRF + self.tagger_language = ENG_SM self.similarity_algo = SpacySimilarity self.similarity_threshold = 0.90 self.chatbot = self._create_chatbot() @@ -85,7 +85,7 @@ class Chatter(Cog): # self.trainer = ListTrainer(self.chatbot) self.config.register_global(**default_global) - self.config.register_guild(**default_guild) + self.config.register_guild(**self.default_guild) self.loop = asyncio.get_event_loop() @@ -371,11 +371,12 @@ class Chatter(Cog): 0: Small 1: Medium 2: Large (Requires additional setup) + 3. Accurate (Requires additional setup) """ - models = [ENG_SM, ENG_MD, ENG_LG] + models = [ENG_SM, ENG_MD, ENG_LG, ENG_TRF] - if model_number < 0 or model_number > 2: + if model_number < 0 or model_number > 3: await ctx.send_help() return diff --git a/chatter/info.json b/chatter/info.json index a9bb96b..0004709 100644 --- a/chatter/info.json +++ b/chatter/info.json @@ -8,6 +8,7 @@ "install_msg": "Thank you for installing Chatter! Please make sure you check the install instructions at https://github.com/bobloy/Fox-V3/blob/master/chatter/README.md\nAfter that, get started ith `[p]load chatter` and `[p]help Chatter`", "requirements": [ "git+git://github.com/bobloy/ChatterBot@fox#egg=ChatterBot", + "en_core_web_sm", "kaggle" ], "short": "Local Chatbot run on machine learning", From 86cc1fa35adb5dc6d8539df67b720d55321bcceb Mon Sep 17 00:00:00 2001 From: bobloy Date: Mon, 5 Jul 2021 14:14:29 -0400 Subject: [PATCH 121/133] Don't specify pyaml --- chatter/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chatter/requirements.txt b/chatter/requirements.txt index 88cd662..490c2a1 100644 --- a/chatter/requirements.txt +++ b/chatter/requirements.txt @@ -3,7 +3,7 @@ mathparse>=0.1,<0.2 nltk>=3.2,<4.0 pint>=0.8.1 python-dateutil>=2.8,<2.9 -pyyaml>=5.3,<5.4 +# pyyaml>=5.3,<5.4 sqlalchemy>=1.3,<1.4 pytz spacy>=2.3,<2.4 From e0a361b952586d25ecc042486ade285c75ca3784 Mon Sep 17 00:00:00 2001 From: bobloy Date: Tue, 6 Jul 2021 10:52:25 -0400 Subject: [PATCH 122/133] remove en_core_web_sm as dependency --- chatter/info.json | 1 - 1 file changed, 1 deletion(-) diff --git a/chatter/info.json b/chatter/info.json index 0004709..a9bb96b 100644 --- a/chatter/info.json +++ b/chatter/info.json @@ -8,7 +8,6 @@ "install_msg": "Thank you for installing Chatter! Please make sure you check the install instructions at https://github.com/bobloy/Fox-V3/blob/master/chatter/README.md\nAfter that, get started ith `[p]load chatter` and `[p]help Chatter`", "requirements": [ "git+git://github.com/bobloy/ChatterBot@fox#egg=ChatterBot", - "en_core_web_sm", "kaggle" ], "short": "Local Chatbot run on machine learning", From 47269ba8f4d8ccd564f73b773fdd35db2ab8e693 Mon Sep 17 00:00:00 2001 From: bobloy Date: Tue, 6 Jul 2021 12:01:27 -0400 Subject: [PATCH 123/133] fix has_table, move age and minutes to trainset --- chatter/chat.py | 10 ++++++++-- chatter/storage_adapters.py | 4 ++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/chatter/chat.py b/chatter/chat.py index e3f14cc..1b81959 100644 --- a/chatter/chat.py +++ b/chatter/chat.py @@ -402,7 +402,13 @@ class Chatter(Cog): ) @commands.is_owner() - @chatter.command(name="minutes") + @chatter.group(name="trainset") + async def chatter_trainset(self, ctx: commands.Context): + """Commands for configuring training""" + pass + + @commands.is_owner() + @chatter_trainset.command(name="minutes") async def minutes(self, ctx: commands.Context, minutes: int): """ Sets the number of minutes the bot will consider a break in a conversation during training @@ -418,7 +424,7 @@ class Chatter(Cog): await ctx.tick() @commands.is_owner() - @chatter.command(name="age") + @chatter_trainset.command(name="age") async def age(self, ctx: commands.Context, days: int): """ Sets the number of days to look back diff --git a/chatter/storage_adapters.py b/chatter/storage_adapters.py index 6f11601..b2dc02a 100644 --- a/chatter/storage_adapters.py +++ b/chatter/storage_adapters.py @@ -5,7 +5,7 @@ class MyDumbSQLStorageAdapter(SQLStorageAdapter): def __init__(self, **kwargs): super(SQLStorageAdapter, self).__init__(**kwargs) - from sqlalchemy import create_engine + from sqlalchemy import create_engine, inspect from sqlalchemy.orm import sessionmaker self.database_uri = kwargs.get("database_uri", False) @@ -31,7 +31,7 @@ class MyDumbSQLStorageAdapter(SQLStorageAdapter): dbapi_connection.execute("PRAGMA journal_mode=WAL") dbapi_connection.execute("PRAGMA synchronous=NORMAL") - if not self.engine.dialect.has_table(self.engine, "Statement"): + if not inspect(self.engine).has_table('Statement'): self.create_database() self.Session = sessionmaker(bind=self.engine, expire_on_commit=True) From b752bfd153507ff54ff999c2d5272da15f6c8ec1 Mon Sep 17 00:00:00 2001 From: bobloy Date: Tue, 6 Jul 2021 13:50:15 -0400 Subject: [PATCH 124/133] Stop looking at DMs --- chatter/chat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chatter/chat.py b/chatter/chat.py index 1b81959..66ff116 100644 --- a/chatter/chat.py +++ b/chatter/chat.py @@ -643,7 +643,7 @@ class Chatter(Cog): guild: discord.Guild = getattr(message, "guild", None) - if await self.bot.cog_disabled_in_guild(self, guild): + if guild is None or await self.bot.cog_disabled_in_guild(self, guild): return ctx: commands.Context = await self.bot.get_context(message) From 1d514f80c696e0b7170cbf168a86862747fab540 Mon Sep 17 00:00:00 2001 From: Brad Duncan <51077147+XargsUK@users.noreply.github.com> Date: Thu, 8 Jul 2021 07:37:37 +0100 Subject: [PATCH 125/133] adding no response exit condition Previously, if no response was provided by the user, the function would continuously loop. I've modified the timeout to 20 seconds. When there's no response, +! to timeoutcount variable. When there's been 3 timeouts, break. --- recyclingplant/recyclingplant.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/recyclingplant/recyclingplant.py b/recyclingplant/recyclingplant.py index cc7bf57..2bf0753 100644 --- a/recyclingplant/recyclingplant.py +++ b/recyclingplant/recyclingplant.py @@ -32,6 +32,7 @@ class RecyclingPlant(Cog): x = 0 reward = 0 + timeoutcount = 0 await ctx.send( "{0} has signed up for a shift at the Recycling Plant! Type ``exit`` to terminate it early.".format( ctx.author.display_name @@ -53,7 +54,7 @@ class RecyclingPlant(Cog): return m.author == ctx.author and m.channel == ctx.channel try: - answer = await self.bot.wait_for("message", timeout=120, check=check) + answer = await self.bot.wait_for("message", timeout=20, check=check) except asyncio.TimeoutError: answer = None From 15ecf72c6483fe8e61467f042dc890b66bc2bf61 Mon Sep 17 00:00:00 2001 From: Brad Duncan <51077147+XargsUK@users.noreply.github.com> Date: Thu, 8 Jul 2021 07:37:45 +0100 Subject: [PATCH 126/133] Update recyclingplant.py --- recyclingplant/recyclingplant.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/recyclingplant/recyclingplant.py b/recyclingplant/recyclingplant.py index 2bf0753..74ff6e8 100644 --- a/recyclingplant/recyclingplant.py +++ b/recyclingplant/recyclingplant.py @@ -59,9 +59,14 @@ class RecyclingPlant(Cog): answer = None if answer is None: - await ctx.send( - "``{}`` fell down the conveyor belt to be sorted again!".format(used["object"]) - ) + if timeoutcount == 2: + await ctx.send( + "{} slacked off at work, so they were sacked with no pay.".format(ctx.author.display_name) + ) + break + else: + await ctx.send("{} is slacking, and if they carry on not working, they'll be fired.".format(ctx.author.display_name)) + timeoutcount += 1 elif answer.content.lower().strip() == used["action"]: await ctx.send( "Congratulations! You put ``{}`` down the correct chute! (**+50**)".format( From 698dafade4cde7c3d691babb37e27430a29be0dd Mon Sep 17 00:00:00 2001 From: bobloy Date: Thu, 8 Jul 2021 08:29:18 -0400 Subject: [PATCH 127/133] Add chatter initialization --- chatter/__init__.py | 6 ++++-- chatter/chat.py | 44 +++++++++++++++++++++++++++++--------------- 2 files changed, 33 insertions(+), 17 deletions(-) diff --git a/chatter/__init__.py b/chatter/__init__.py index 9447c6a..663dadf 100644 --- a/chatter/__init__.py +++ b/chatter/__init__.py @@ -1,8 +1,10 @@ from .chat import Chatter -def setup(bot): - bot.add_cog(Chatter(bot)) +async def setup(bot): + cog = Chatter(bot) + await cog.initialize() + bot.add_cog(cog) # __all__ = ( diff --git a/chatter/chat.py b/chatter/chat.py index 66ff116..4655da8 100644 --- a/chatter/chat.py +++ b/chatter/chat.py @@ -19,6 +19,7 @@ from redbot.core.utils.predicates import MessagePredicate from chatter.trainers import MovieTrainer, TwitterCorpusTrainer, UbuntuCorpusTrainer2 +chatterbot_log = logging.getLogger("red.fox_v3.chatterbot") log = logging.getLogger("red.fox_v3.chatter") @@ -58,11 +59,14 @@ class Chatter(Cog): This cog trains a chatbot that will talk like members of your Guild """ + models = [ENG_SM, ENG_MD, ENG_LG, ENG_TRF] + algos = [SpacySimilarity, JaccardSimilarity, LevenshteinDistance] + def __init__(self, bot): super().__init__() self.bot = bot self.config = Config.get_conf(self, identifier=6710497116116101114) - default_global = {"learning": True} + default_global = {"learning": True, "model_number": 0, "algo_number": 0, "threshold": 0.90} self.default_guild = { "whitelist": None, "days": 1, @@ -79,7 +83,7 @@ class Chatter(Cog): self.tagger_language = ENG_SM self.similarity_algo = SpacySimilarity self.similarity_threshold = 0.90 - self.chatbot = self._create_chatbot() + self.chatbot = None # self.chatbot.set_trainer(ListTrainer) # self.trainer = ListTrainer(self.chatbot) @@ -98,6 +102,18 @@ class Chatter(Cog): """Nothing to delete""" return + async def initialize(self): + all_config = dict(self.config.defaults["GLOBAL"]) + all_config.update(await self.config.all()) + model_number = all_config["model_number"] + algo_number = all_config["algo_number"] + threshold = all_config["threshold"] + + self.tagger_language = self.models[model_number] + self.similarity_algo = self.algos[algo_number] + self.similarity_threshold = threshold + self.chatbot = self._create_chatbot() + def _create_chatbot(self): return ChatBot( @@ -110,7 +126,7 @@ class Chatter(Cog): logic_adapters=["chatterbot.logic.BestMatch"], maximum_similarity_threshold=self.similarity_threshold, tagger_language=self.tagger_language, - logger=log, + logger=chatterbot_log, ) async def _get_conversation(self, ctx, in_channels: List[discord.TextChannel]): @@ -334,15 +350,12 @@ class Chatter(Cog): self, ctx: commands.Context, algo_number: int, threshold: float = None ): """ - Switch the active logic algorithm to one of the three. Default after reload is Spacy + Switch the active logic algorithm to one of the three. Default is Spacy 0: Spacy 1: Jaccard 2: Levenshtein """ - - algos = [SpacySimilarity, JaccardSimilarity, LevenshteinDistance] - if algo_number < 0 or algo_number > 2: await ctx.send_help() return @@ -355,8 +368,11 @@ class Chatter(Cog): return else: self.similarity_threshold = threshold + await self.config.threshold.set(self.similarity_threshold) + + self.similarity_algo = self.algos[algo_number] + await self.config.algo_number.set(algo_number) - self.similarity_algo = algos[algo_number] async with ctx.typing(): self.chatbot = self._create_chatbot() @@ -366,21 +382,18 @@ class Chatter(Cog): @chatter.command(name="model") async def chatter_model(self, ctx: commands.Context, model_number: int): """ - Switch the active model to one of the three. Default after reload is Medium + Switch the active model to one of the three. Default is Small 0: Small - 1: Medium + 1: Medium (Requires additional setup) 2: Large (Requires additional setup) 3. Accurate (Requires additional setup) """ - - models = [ENG_SM, ENG_MD, ENG_LG, ENG_TRF] - if model_number < 0 or model_number > 3: await ctx.send_help() return - if model_number == 2: + if model_number >= 0: await ctx.maybe_send_embed( "Additional requirements needed. See guide before continuing.\n" "Continue?" ) @@ -393,7 +406,8 @@ class Chatter(Cog): if not pred.result: return - self.tagger_language = models[model_number] + self.tagger_language = self.models[model_number] + await self.config.model_number.set(model_number) async with ctx.typing(): self.chatbot = self._create_chatbot() From 3eb499bf0e0dd3dbc72553fcef39d992d359592b Mon Sep 17 00:00:00 2001 From: bobloy Date: Thu, 8 Jul 2021 08:30:21 -0400 Subject: [PATCH 128/133] black reformat --- recyclingplant/recyclingplant.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/recyclingplant/recyclingplant.py b/recyclingplant/recyclingplant.py index 74ff6e8..3751648 100644 --- a/recyclingplant/recyclingplant.py +++ b/recyclingplant/recyclingplant.py @@ -61,11 +61,17 @@ class RecyclingPlant(Cog): if answer is None: if timeoutcount == 2: await ctx.send( - "{} slacked off at work, so they were sacked with no pay.".format(ctx.author.display_name) + "{} slacked off at work, so they were sacked with no pay.".format( + ctx.author.display_name ) + ) break else: - await ctx.send("{} is slacking, and if they carry on not working, they'll be fired.".format(ctx.author.display_name)) + await ctx.send( + "{} is slacking, and if they carry on not working, they'll be fired.".format( + ctx.author.display_name + ) + ) timeoutcount += 1 elif answer.content.lower().strip() == used["action"]: await ctx.send( From a5ff888f4c19c1297a47f15c04f9891cb4e29c7d Mon Sep 17 00:00:00 2001 From: bobloy Date: Thu, 8 Jul 2021 08:33:29 -0400 Subject: [PATCH 129/133] black reformat --- chatter/storage_adapters.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/chatter/storage_adapters.py b/chatter/storage_adapters.py index b2dc02a..706f96f 100644 --- a/chatter/storage_adapters.py +++ b/chatter/storage_adapters.py @@ -18,9 +18,7 @@ class MyDumbSQLStorageAdapter(SQLStorageAdapter): if not self.database_uri: self.database_uri = "sqlite:///db.sqlite3" - self.engine = create_engine( - self.database_uri, connect_args={"check_same_thread": False} - ) + self.engine = create_engine(self.database_uri, connect_args={"check_same_thread": False}) if self.database_uri.startswith("sqlite://"): from sqlalchemy.engine import Engine @@ -31,7 +29,7 @@ class MyDumbSQLStorageAdapter(SQLStorageAdapter): dbapi_connection.execute("PRAGMA journal_mode=WAL") dbapi_connection.execute("PRAGMA synchronous=NORMAL") - if not inspect(self.engine).has_table('Statement'): + if not inspect(self.engine).has_table("Statement"): self.create_database() self.Session = sessionmaker(bind=self.engine, expire_on_commit=True) From b2e843e78153c2f73759f7650a4ac6aa630e6bb3 Mon Sep 17 00:00:00 2001 From: bobloy Date: Thu, 15 Jul 2021 14:57:33 -0400 Subject: [PATCH 130/133] Invalid timestring instead of error --- timerole/timerole.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/timerole/timerole.py b/timerole/timerole.py index 0d62cf0..f0eb716 100644 --- a/timerole/timerole.py +++ b/timerole/timerole.py @@ -92,6 +92,9 @@ class Timerole(Cog): await ctx.maybe_send_embed("Error: Invalid time string.") return + if parsed_time is None: + return await ctx.maybe_send_embed("Error: Invalid time string.") + days = parsed_time.days hours = parsed_time.seconds // 60 // 60 From 2421c4e9bf8c707ed79de0c07e1c2ad5ddf129dc Mon Sep 17 00:00:00 2001 From: bobloy Date: Thu, 15 Jul 2021 15:09:41 -0400 Subject: [PATCH 131/133] Add sm and md to requirements --- chatter/info.json | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/chatter/info.json b/chatter/info.json index a9bb96b..27db873 100644 --- a/chatter/info.json +++ b/chatter/info.json @@ -8,7 +8,9 @@ "install_msg": "Thank you for installing Chatter! Please make sure you check the install instructions at https://github.com/bobloy/Fox-V3/blob/master/chatter/README.md\nAfter that, get started ith `[p]load chatter` and `[p]help Chatter`", "requirements": [ "git+git://github.com/bobloy/ChatterBot@fox#egg=ChatterBot", - "kaggle" + "kaggle", + "https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.1.0/en_core_web_sm-3.1.0.tar.gz#egg=en_core_web_sm", + "https://github.com/explosion/spacy-models/releases/download/en_core_web_md-3.1.0/en_core_web_md-3.1.0.tar.gz#egg=en_core_web_md" ], "short": "Local Chatbot run on machine learning", "end_user_data_statement": "This cog only stores anonymous conversations data; no End User Data is stored.", From 41836073727070fa4fae0b7f8c69159a577c8513 Mon Sep 17 00:00:00 2001 From: bobloy Date: Thu, 15 Jul 2021 16:05:39 -0400 Subject: [PATCH 132/133] Add option for skipping bots --- timerole/timerole.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/timerole/timerole.py b/timerole/timerole.py index f0eb716..14a0bb4 100644 --- a/timerole/timerole.py +++ b/timerole/timerole.py @@ -37,7 +37,7 @@ class Timerole(Cog): self.bot = bot self.config = Config.get_conf(self, identifier=9811198108111121, force_registration=True) default_global = {} - default_guild = {"announce": None, "reapply": True, "roles": {}} + default_guild = {"announce": None, "reapply": True, "roles": {}, "skipbots": True} default_rolemember = {"had_role": False, "check_again_time": None} self.config.register_global(**default_global) @@ -154,6 +154,14 @@ class Timerole(Cog): await self.config.guild(guild).reapply.set(not current_setting) await ctx.maybe_send_embed(f"Reapplying roles is now set to: {not current_setting}") + @timerole.command() + async def skipbots(self, ctx: commands.Context): + """Toggle skipping bots when adding/removing roles. Defaults to True""" + guild = ctx.guild + current_setting = await self.config.guild(guild).skipbots() + await self.config.guild(guild).skipbots.set(not current_setting) + await ctx.maybe_send_embed(f"Skipping bots is now set to: {not current_setting}") + @timerole.command() async def delrole(self, ctx: commands.Context, role: discord.Role): """Deletes a role from being added/removed after specified time""" @@ -202,6 +210,7 @@ class Timerole(Cog): remove_results = "" reapply = all_guilds[guild_id]["reapply"] role_dict = all_guilds[guild_id]["roles"] + skipbots = all_guilds[guild_id]["skipbots"] if not any(role_dict.values()): # No roles log.debug(f"No roles are configured for guild: {guild}") @@ -211,6 +220,10 @@ class Timerole(Cog): # log.debug(f"{all_mr=}") async for member in AsyncIter(guild.members, steps=10): + + if member.bot and skipbots: + continue + addlist = [] removelist = [] From 61fa006e337a4d09299cc83e683413ca701cb3d9 Mon Sep 17 00:00:00 2001 From: bobloy Date: Fri, 16 Jul 2021 09:26:06 -0400 Subject: [PATCH 133/133] QoL fixes --- conquest/conquest.py | 11 ++++++++++- conquest/data/assets/maps.json | 6 +++--- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/conquest/conquest.py b/conquest/conquest.py index fa70911..2c3772c 100644 --- a/conquest/conquest.py +++ b/conquest/conquest.py @@ -1,5 +1,6 @@ import asyncio import json +import logging import os import pathlib from abc import ABC @@ -13,6 +14,8 @@ from redbot.core import Config, commands from redbot.core.bot import Red from redbot.core.data_manager import bundled_data_path, cog_data_path +log = logging.getLogger("red.fox_v3.conquest") + class Conquest(commands.Cog): """ @@ -53,14 +56,20 @@ class Conquest(commands.Cog): self.current_map = await self.config.current_map() if self.current_map: - await self.current_map_load() + if not await self.current_map_load(): + await self.config.current_map.clear() async def current_map_load(self): map_data_path = self.asset_path / self.current_map / "data.json" + if not map_data_path.exists(): + log.warning(f"{map_data_path} does not exist. Clearing current map") + return False + with map_data_path.open() as mapdata: self.map_data: dict = json.load(mapdata) self.ext = self.map_data["extension"] self.ext_format = "JPEG" if self.ext.upper() == "JPG" else self.ext.upper() + return True @commands.group() async def conquest(self, ctx: commands.Context): diff --git a/conquest/data/assets/maps.json b/conquest/data/assets/maps.json index a7d1c03..671a807 100644 --- a/conquest/data/assets/maps.json +++ b/conquest/data/assets/maps.json @@ -1,7 +1,7 @@ { "maps": [ - "simple_blank_map", - "test", - "test2" + "simple", + "ck2", + "HoI" ] } \ No newline at end of file