tiktoken does not work on replit

Support Bug Reports
1

Here is the code written by Ghostwriter:

import time

import requests
import tiktoken

file_url = "https://replit.com/@PerspectivasBri/NH-Python#PROMPTS1.txt"

def tokenize_prompt(prompt):
    tokenizer = tiktoken.get_encoding("cl100k_base")
    prompt_tokens = tokenizer.encode(prompt)
    return prompt_tokens

def read_file(file_url):
    response = requests.get(file_url)
    content = response.text
    return content.split("\n")

def encode_prompts(prompts):
    encoded_prompts = []
    for prompt in prompts:
        prompt_tokens = tokenize_prompt(prompt)
        encoded_prompt = " ".join(str(token) for token in prompt_tokens)
        encoded_prompts.append(encoded_prompt)
    return encoded_prompts

def gpt3_turbo_encode(file_url):
    prompts = read_file(file_url)
    encoded_prompts = encode_prompts(prompts)
    encoded_text = "\n".join(encoded_prompts)
  
    # Store the encoded prompts back to the same URL
    requests.post(file_url, data=encoded_text)
    
    # Progress Report
    while True:
        print("Progress report...")
        time.sleep(60)  # Wait for 60 seconds before printing the next progress report

gpt3_turbo_encode(file_url)

replit resources apparently are all working OK, without errors, but after 30 minutes: NO output.
I should be seeing a report on my console every 60 seconds.
Can you throw me a clue?

Expected vs Current Behavior:

Steps to reproduce:

Bug appears at this link: https://replit.com/@PerspectivasBri/NH-Python#main1.py

Screenshot(s)/Screen Recording:

Browser/OS/Device: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36

Replit Profile: https://replit.com/@PerspectivasBri

2

Hey @PerspectivasBri welcome to the forums!

Is your Repl private? Is this your full code? Can you put it all in backticks

```py
#code here
```
1 Like
3 
import time

import requests
import tiktoken

file_url = "https://replit.com/@PerspectivasBri/NH-Python#PROMPTS1.txt"

def tokenize_prompt(prompt):
    tokenizer = tiktoken.get_encoding("cl100k_base")
    prompt_tokens = tokenizer.encode(prompt)
    return prompt_tokens

def read_file(file_url):
    response = requests.get(file_url)
    content = response.text
    return content.split("\n")

def encode_prompts(prompts):
    encoded_prompts = []
    for prompt in prompts:
        prompt_tokens = tokenize_prompt(prompt)
        encoded_prompt = " ".join(str(token) for token in prompt_tokens)
        encoded_prompts.append(encoded_prompt)
    return encoded_prompts

def gpt3_turbo_encode(file_url):
    prompts = read_file(file_url)
    encoded_prompts = encode_prompts(prompts)
    encoded_text = "\n".join(encoded_prompts)
  
    # Store the encoded prompts back to the same URL
    requests.post(file_url, data=encoded_text)
    
    # Progress Report
    while True:
        print("Progress report...")
        time.sleep(60)  # Wait for 60 seconds before printing the next progress report

gpt3_turbo_encode(file_url)
4

NH-Python is Public

Please try to reproduce this issue on another account.

Thanks!

5

@PerspectivasBri use backticks not quotes (backticks can be found below the tilda ~)

6 
import time

import requests
import tiktoken

file_url = "https://replit.com/@PerspectivasBri/NH-Python#PROMPTS1.txt"

def tokenize_prompt(prompt):
    tokenizer = tiktoken.get_encoding("cl100k_base")
    prompt_tokens = tokenizer.encode(prompt)
    return prompt_tokens

def read_file(file_url):
    response = requests.get(file_url)
    content = response.text
    return content.split("\n")

def encode_prompts(prompts):
    encoded_prompts = []
    for prompt in prompts:
        prompt_tokens = tokenize_prompt(prompt)
        encoded_prompt = " ".join(str(token) for token in prompt_tokens)
        encoded_prompts.append(encoded_prompt)
    return encoded_prompts

def gpt3_turbo_encode(file_url):
    prompts = read_file(file_url)
    encoded_prompts = encode_prompts(prompts)
    encoded_text = "\n".join(encoded_prompts)
  
    # Store the encoded prompts back to the same URL
    requests.post(file_url, data=encoded_text)
    
    # Progress Report
    while True:
        print("Progress report...")
        time.sleep(60)  # Wait for 60 seconds before printing the next progress report

gpt3_turbo_encode(file_url)