feat: update model tokens

Author: VinciGit00

scrapegraphai/helpers/models_tokens.py

@@ -24,20 +24,37 @@
         "gpt-4o-2024-08-06": 128000,
         "gpt-4o-2024-05-13": 128000,
         "gpt-4o-mini": 128000,
-        "gpt-4.1": 1000000,
-        "gpt-4.1-mini": 1000000,
-        "gpt-4.1-nano": 1000000,
+        "gpt-4.1": 1048576,
+        "gpt-4.1-mini": 1048576,
+        "gpt-4.1-nano": 1048576,
         "gpt-4.5": 128000,
         "gpt-4.5-preview": 128000,
-        "o1-preview": 128000,
+        "o1-preview": 200000,
         "o1-mini": 128000,
-        "o1": 128000,
+        "o1": 200000,
+        "o1-pro": 200000,
         "o3-mini": 200000,
+        "o3": 200000,
+        "o3-pro": 200000,
+        "o4-mini": 200000,
+        "o3-deep-research": 200000,
+        "o4-mini-deep-research": 200000,
+        "gpt-5": 200000,
+        "gpt-5.1": 200000,
         "gpt-5.2": 128000,
         "gpt-5.2-pro": 128000,
+        "gpt-5.2-codex": 128000,
+        "gpt-5.1-codex": 200000,
+        "gpt-5.1-codex-max": 200000,
+        "gpt-5.1-codex-mini": 200000,
+        "gpt-5-codex": 200000,
         "gpt-5.2-chat-latest": 128000,
+        "gpt-5.1-chat-latest": 200000,
+        "gpt-5-chat-latest": 200000,
         "gpt-5-mini": 128000,
         "gpt-5-nano": 128000,
+        "gpt-oss-120b": 128000,
+        "gpt-oss-20b": 128000,
     },
     "azure_openai": {
         "gpt-3.5-turbo-0125": 16385,
@@ -58,13 +75,33 @@
         "gpt-4o": 128000,
         "gpt-4o-mini": 128000,
         "chatgpt-4o-latest": 128000,
-        "o1-preview": 128000,
+        "o1-preview": 200000,
         "o1-mini": 128000,
+        "o1": 200000,
+        "o1-pro": 200000,
+        "o3-mini": 200000,
+        "o3": 200000,
+        "o3-pro": 200000,
+        "o4-mini": 200000,
+        "gpt-4.1": 1048576,
+        "gpt-4.1-mini": 1048576,
+        "gpt-4.1-nano": 1048576,
+        "gpt-5": 200000,
+        "gpt-5.1": 200000,
         "gpt-5.2": 128000,
         "gpt-5.2-pro": 128000,
+        "gpt-5.2-codex": 128000,
+        "gpt-5.1-codex": 200000,
+        "gpt-5.1-codex-max": 200000,
+        "gpt-5.1-codex-mini": 200000,
+        "gpt-5-codex": 200000,
         "gpt-5.2-chat-latest": 128000,
+        "gpt-5.1-chat-latest": 200000,
+        "gpt-5-chat-latest": 200000,
         "gpt-5-mini": 128000,
         "gpt-5-nano": 128000,
+        "gpt-oss-120b": 128000,
+        "gpt-oss-20b": 128000,
     },
     "google_genai": {
         "gemini-pro": 128000,
@@ -79,6 +116,10 @@
         "gemini-1.5-flash": 128000,
         "gemini-1.5-pro": 128000,
         "gemini-1.0-pro": 128000,
+        "gemini-2.0-flash": 1048576,
+        "gemini-2.0-flash-exp": 1048576,
+        "gemini-2.0-pro": 2000000,
+        "gemini-2.0-pro-exp": 2000000,
     },
     "ollama": {
         "command-r": 12800,
@@ -96,7 +137,7 @@
         "llama3.1": 128000,
         "llama3.1:8b": 128000,
         "llama3.1:70b": 128000,
-        "lama3.1:405b": 128000,
+        "llama3.1:405b": 128000,
         "llama3.2": 128000,
         "llama3.2:1b": 128000,
         "llama3.2:3b": 128000,
@@ -139,8 +180,8 @@
         "qwen-turbo": 6000,
     },
     "nvidia": {
-        "meta/llama3-70b-instruct": 419,
-        "meta/llama3-8b-instruct": 419,
+        "meta/llama3-70b-instruct": 8192,
+        "meta/llama3-8b-instruct": 8192,
         "nemotron-4-340b-instruct": 1024,
         "databricks/dbrx-instruct": 4096,
         "google/codegemma-7b": 8192,
@@ -155,6 +196,7 @@
         "mistralai/mixtral-8x22b-instruct-v0.1": 32768,
         "mistralai/mixtral-8x7b-instruct-v0.1": 8192,
         "snowflake/arctic": 16384,
+        "meta/llama-3.3-70b-instruct": 128000,
     },
     "groq": {
         "llama3-8b-8192": 8192,