So, I am wanting to make all the lines in a text file send a request to the twitch login api to get their IDS, now there is about 10k lines in this text file and I want to send them in parallel but my current code stops the code execution at print(“Sending Request”) and then it moves onto the next line in the text file.
If I run the code one at a time like asyncio.run() it works just fine but this is really slow. Anyone I can make these requests run parallel?
async def get_id(sem, username, oauth):
"""
Hit the login endpoint so we can get the ID of each account we have. This will allow us to use the new twitch api.
"""
async with sem:
Oauth = oauth
oauth = f"Bearer {oauth}"
async with aiohttp.ClientSession() as session:
try:
print("Sending Request")
get_userID = await session.get('https://api.twitch.tv/helix/users?login=' + username, headers={'Client-Id':'kimne78kx3ncx6brgo4mv6wki5h1ko', "Authorization": oauth})
parse_response = await get_userID.json()
print("Got Response")
USERID = parse_response["data"][0]["id"]
print("Got USERID")
print(f"Username: {username} | ID: {USERID} | Oauth: {Oauth}")
"""
This function writes our ID and some other useful info from the twitch api to the file.
"""
cwd = os.getcwd()
async with aiofiles.open(f"{cwd}/accounts/working_accounts.txt", mode="a") as accountFile:
await accountFile.write(f"{username}:{USERID}:{Oauth}\n")
except:
pass
async def run():
sem = asyncio.Semaphore(limit)
cwd = os.getcwd()
ACCOUNTS = await read_file(f"{cwd}/accounts/accounts.txt")
"""
Our for loop to make a new coroutine for each line in our text file. (approx 10k)
"""
tasks = []
for account in ACCOUNTS:
formatAccount = account.split(":")
username = formatAccount[0].replace("\n", "")
oauth = formatAccount[1].replace("\n", "")
task = asyncio.ensure_future(get_id(sem, username, oauth))
tasks.append(task)
response = await asyncio.gather(*tasks)
if __name__ == "__main__":
loop = asyncio.get_event_loop()
loop.run_until_complete(asyncio.ensure_future(run()))