400 lines
14 KiB
Python
400 lines
14 KiB
Python
from math import sqrt, cbrt
|
|
from enum import Enum
|
|
from typing import NamedTuple
|
|
from urllib.parse import quote
|
|
from datetime import datetime, timedelta
|
|
from time import sleep
|
|
|
|
class CardType(Enum):
|
|
other = 0
|
|
person = 1
|
|
location = 2
|
|
artwork = 3
|
|
life_form = 4
|
|
conflict = 5
|
|
group = 6
|
|
science_thing = 7
|
|
vehicle = 8
|
|
|
|
class CardRarity(Enum):
|
|
common = 0
|
|
uncommon = 1
|
|
rare = 2
|
|
super_rare = 3
|
|
epic = 4
|
|
legendary = 5
|
|
|
|
class Card(NamedTuple):
|
|
name: str
|
|
created_at: datetime
|
|
image_link: str
|
|
card_rarity: CardRarity
|
|
card_type: CardType
|
|
wikidata_instance: str
|
|
text: str
|
|
attack: int
|
|
defense: int
|
|
cost: int
|
|
|
|
def __str__(self) -> str:
|
|
return_string = "┏"+"━"*50+"┓\n"
|
|
return_string += "┃"+f"{self.name:{' '}<50}"+"┃\n"
|
|
return_string += "┠"+"─"*50+"┨\n"
|
|
card_type = "Type: "+self.card_type.name
|
|
if self.card_type == CardType.other:
|
|
card_type += f" ({self.wikidata_instance})"
|
|
return_string += "┃"+f"{card_type:{' '}<50}"+"┃\n"
|
|
return_string += "┠"+"─"*50+"┨\n"
|
|
rarity_text = f"Rarity: {self.card_rarity.name}"
|
|
return_string += "┃"+f"{rarity_text:<50}"+"┃\n"
|
|
return_string += "┠"+"─"*50+"┨\n"
|
|
link = "Image: "+("Yes" if self.image_link != "" else "No")
|
|
return_string += "┃"+f"{link:{' '}<50}"+"┃\n"
|
|
return_string += "┠"+"─"*50+"┨\n"
|
|
lines = []
|
|
words = self.text.split(" ")
|
|
current_line = ""
|
|
for w in words:
|
|
if len(current_line + " " + w) < 50:
|
|
current_line += " " + w
|
|
elif len(current_line) < 40 and len(w) > 10:
|
|
new_len = 48 - len(current_line)
|
|
current_line += " " + w[:new_len] + "-"
|
|
lines.append(current_line)
|
|
current_line = w[new_len:]
|
|
else:
|
|
lines.append(current_line)
|
|
current_line = w
|
|
|
|
lines.append(current_line)
|
|
|
|
for l in lines :
|
|
return_string += "┃"+f"{l:{' '}<50}"+"┃\n"
|
|
return_string += "┠"+"─"*50+"┨\n"
|
|
|
|
date_text = str(self.created_at.date())
|
|
stats = f"{self.attack}/{self.defense}"
|
|
spaces = 50 - (len(date_text) + len(stats))
|
|
return_string += "┃"+date_text + " "*spaces + stats + "┃\n"
|
|
return_string += "┗"+"━"*50+"┛"
|
|
return return_string
|
|
|
|
WIKIDATA_INSTANCE_TYPE_MAP = {
|
|
"Q5": CardType.person, # human
|
|
"Q15632617": CardType.person, # fictional human
|
|
"Q215627": CardType.person, # person
|
|
|
|
"Q7889": CardType.artwork, # video game
|
|
"Q1004": CardType.artwork, # comics
|
|
"Q15416": CardType.artwork, # television program
|
|
"Q11424": CardType.artwork, # film
|
|
"Q24862": CardType.artwork, # film
|
|
"Q11032": CardType.artwork, # newspaper
|
|
"Q25379": CardType.artwork, # play
|
|
"Q482994": CardType.artwork, # album
|
|
"Q134556": CardType.artwork, # single
|
|
"Q169930": CardType.artwork, # EP
|
|
"Q202866": CardType.artwork, # animated film
|
|
"Q734698": CardType.artwork, # collectible card game
|
|
"Q506240": CardType.artwork, # television film
|
|
"Q738377": CardType.artwork, # student newspaper
|
|
"Q3305213": CardType.artwork, # painting
|
|
"Q3177859": CardType.artwork, # dedicated deck card game
|
|
"Q5398426": CardType.artwork, # television series
|
|
"Q7725634": CardType.artwork, # literary work
|
|
"Q1761818": CardType.artwork, # advertising campaign
|
|
"Q1446621": CardType.artwork, # recital
|
|
"Q63952888": CardType.artwork, # anime television series
|
|
"Q47461344": CardType.artwork, # written work
|
|
"Q71631512": CardType.artwork, # tabletop role-playing game supplement
|
|
"Q21198342": CardType.artwork, # manga series
|
|
"Q24634210": CardType.artwork, # podcast show
|
|
"Q105543609": CardType.artwork, # musical work / composition
|
|
"Q106499608": CardType.artwork, # literary reading
|
|
|
|
"Q515": CardType.location, # city
|
|
"Q8502": CardType.location, # mountain
|
|
"Q4022": CardType.location, # river
|
|
"Q6256": CardType.location, # country
|
|
"Q41176": CardType.location, # building
|
|
"Q23442": CardType.location, # island
|
|
"Q82794": CardType.location, # geographic region
|
|
"Q34442": CardType.location, # road
|
|
"Q3624078": CardType.location, # sovereign state
|
|
"Q1093829": CardType.location, # city in the United States
|
|
"Q7930989": CardType.location, # city/town
|
|
"Q35145263": CardType.location, # natural geographic object
|
|
|
|
"Q16521": CardType.life_form, # taxon
|
|
"Q23038290": CardType.life_form, # fossil taxon
|
|
|
|
"Q198": CardType.conflict, # war
|
|
"Q8465": CardType.conflict, # civil war
|
|
"Q103495": CardType.conflict, # world war
|
|
"Q997267": CardType.conflict, # skirmish
|
|
"Q178561": CardType.conflict, # battle
|
|
"Q1361229": CardType.conflict, # conquest
|
|
|
|
"Q7278": CardType.group, # political party
|
|
"Q476028": CardType.group, # association football club
|
|
"Q215380": CardType.group, # musical group
|
|
"Q176799": CardType.group, # military unit
|
|
"Q178790": CardType.group, # labor union
|
|
"Q2367225": CardType.group, # university and college sports club
|
|
"Q9248092": CardType.group, # infantry division
|
|
"Q7210356": CardType.group, # political organization
|
|
"Q5419137": CardType.group, # veterans' organization
|
|
"Q12973014": CardType.group, # sports team
|
|
"Q135408445": CardType.group, # men's national association football team
|
|
|
|
"Q7187": CardType.science_thing, # gene
|
|
"Q8054": CardType.science_thing, # protein
|
|
"Q11276": CardType.science_thing, # globular cluster
|
|
"Q898273": CardType.science_thing, # protein domain
|
|
"Q168845": CardType.science_thing, # star cluster
|
|
"Q113145171": CardType.science_thing, # type of chemical entity
|
|
|
|
"Q43193": CardType.vehicle, # truck
|
|
"Q25956": CardType.vehicle, # space station
|
|
"Q174736": CardType.vehicle, # destroyer
|
|
"Q484000": CardType.vehicle, # unmanned aerial vehicle
|
|
"Q559026": CardType.vehicle, # ship class
|
|
"Q3231690": CardType.vehicle, # automobile model
|
|
"Q1428357": CardType.vehicle, # submarine class
|
|
"Q1499623": CardType.vehicle, # destroyer escort
|
|
}
|
|
|
|
import asyncio
|
|
import httpx
|
|
|
|
HEADERS = {"User-Agent": "WikiTCG/1.0 (nikolaj@gade.gg)"}
|
|
|
|
async def _get_random_summary_async(client: httpx.AsyncClient) -> dict:
|
|
try:
|
|
response = await client.get(
|
|
"https://en.wikipedia.org/api/rest_v1/page/random/summary",
|
|
headers=HEADERS,
|
|
follow_redirects=False,
|
|
)
|
|
if response.status_code in (301, 302, 303, 307, 308):
|
|
# Extract the title from the redirect location and re-encode it
|
|
location = response.headers["location"]
|
|
title = location.split("/page/summary/")[-1]
|
|
response = await client.get(
|
|
f"https://en.wikipedia.org/api/rest_v1/page/summary/{quote(title, safe='%')}",
|
|
headers=HEADERS,
|
|
follow_redirects=False,
|
|
)
|
|
except httpx.ReadTimeout:
|
|
return {}
|
|
|
|
if not response.is_success:
|
|
print("Error in request:")
|
|
print(response.status_code)
|
|
print(response.text)
|
|
return {}
|
|
|
|
return response.json()
|
|
|
|
async def _get_page_summary_async(client: httpx.AsyncClient, title: str) -> dict:
|
|
try:
|
|
response = await client.get(
|
|
f"https://en.wikipedia.org/api/rest_v1/page/summary/{title}",
|
|
headers=HEADERS,
|
|
follow_redirects=False,
|
|
)
|
|
if response.status_code in (301, 302, 303, 307, 308):
|
|
# Extract the title from the redirect location and re-encode it
|
|
location = response.headers["location"]
|
|
title = location.split("/page/summary/")[-1]
|
|
response = await client.get(
|
|
f"https://en.wikipedia.org/api/rest_v1/page/summary/{quote(title, safe='%')}",
|
|
headers=HEADERS,
|
|
follow_redirects=False,
|
|
)
|
|
except httpx.ReadTimeout:
|
|
return {}
|
|
|
|
if not response.is_success:
|
|
print("Error in request:")
|
|
print(response.status_code)
|
|
print(response.text)
|
|
return {}
|
|
|
|
return response.json()
|
|
|
|
async def _infer_card_type_async(client: httpx.AsyncClient, entity_id: str) -> tuple[CardType, str, int]:
|
|
response = await client.get(
|
|
"https://www.wikidata.org/wiki/Special:EntityData/" + entity_id + ".json",
|
|
headers=HEADERS
|
|
)
|
|
if not response.is_success:
|
|
return CardType.other, "", 0
|
|
|
|
entity = response.json().get("entities", {}).get(entity_id, {})
|
|
claims = entity.get("claims", {})
|
|
instance_of_claims = claims.get("P31", [])
|
|
qids = [c.get("mainsnak", {}).get("datavalue", {}).get("value", {}).get("id") for c in instance_of_claims]
|
|
|
|
sitelinks = entity.get("sitelinks", {})
|
|
language_count = sum(
|
|
1 for key in sitelinks
|
|
if key.endswith("wiki")
|
|
and key not in ("commonswiki", "wikidatawiki", "specieswiki")
|
|
)
|
|
|
|
if "P625" in claims:
|
|
return CardType.location, (qids[0] if qids != [] else ""), language_count
|
|
|
|
for qid in qids:
|
|
if qid in WIKIDATA_INSTANCE_TYPE_MAP:
|
|
return WIKIDATA_INSTANCE_TYPE_MAP[qid], qid, language_count
|
|
return CardType.other, (qids[0] if qids != [] else ""), language_count
|
|
|
|
async def _get_wikirank_score(client: httpx.AsyncClient, title: str) -> float | None:
|
|
"""Returns a quality score from 0-100, or None if unavailable."""
|
|
response = await client.get(
|
|
f"https://api.wikirank.net/api.php?name={quote(title, safe="")}&lang=en",
|
|
headers=HEADERS
|
|
)
|
|
if not response.is_success:
|
|
return None
|
|
data = response.json()
|
|
result = data.get("result",{})
|
|
if result == "not found":
|
|
return None
|
|
return result.get("en",{}).get("quality")
|
|
|
|
def _score_to_rarity(score: float | None) -> CardRarity:
|
|
if score is None:
|
|
return CardRarity.common
|
|
if score >= 95:
|
|
return CardRarity.legendary
|
|
if score >= 80:
|
|
return CardRarity.epic
|
|
if score >= 65:
|
|
return CardRarity.super_rare
|
|
if score >= 50:
|
|
return CardRarity.rare
|
|
if score >= 35:
|
|
return CardRarity.uncommon
|
|
return CardRarity.common
|
|
|
|
RARITY_MULTIPLIER = {
|
|
CardRarity.common: 1,
|
|
CardRarity.uncommon: 1.1,
|
|
CardRarity.rare: 1.3,
|
|
CardRarity.super_rare: 1.7,
|
|
CardRarity.epic: 2.5,
|
|
CardRarity.legendary: 3,
|
|
}
|
|
|
|
async def _get_monthly_pageviews(client: httpx.AsyncClient, title: str) -> int | None:
|
|
# Uses the last full month
|
|
today = datetime.now()
|
|
first_of_month = today.replace(day=1)
|
|
last_month_end = first_of_month - timedelta(days=1)
|
|
last_month_start = last_month_end.replace(day=1)
|
|
start = last_month_start.strftime("%Y%m01")
|
|
end = last_month_end.strftime("%Y%m%d")
|
|
|
|
try:
|
|
response = await client.get(
|
|
f"https://wikimedia.org/api/rest_v1/metrics/pageviews/per-article/en.wikipedia/all-access/all-agents/{quote(title, safe='%')}/monthly/{start}/{end}",
|
|
headers=HEADERS,
|
|
)
|
|
if not response.is_success:
|
|
return None
|
|
items = response.json().get("items", [])
|
|
return items[0]["views"] if items else None
|
|
except httpx.ReadError:
|
|
return None
|
|
|
|
def _pageviews_to_defense(views: int | None) -> int:
|
|
if views is None:
|
|
return 0
|
|
return int(sqrt(views))
|
|
|
|
async def _get_card_async(client: httpx.AsyncClient, page_title: str|None = None) -> Card|None:
|
|
if page_title is None:
|
|
summary = await _get_random_summary_async(client)
|
|
else:
|
|
summary = await _get_page_summary_async(client, quote(page_title))
|
|
if summary == {}:
|
|
return None
|
|
|
|
title = summary["title"]
|
|
entity_id = summary.get("wikibase_item")
|
|
text = summary.get("extract", "")
|
|
|
|
card_type_task = (
|
|
_infer_card_type_async(client, entity_id)
|
|
if entity_id
|
|
else asyncio.sleep(0, result=(CardType.other, "", 0))
|
|
)
|
|
wikirank_task = _get_wikirank_score(client, title)
|
|
pageviews_task = _get_monthly_pageviews(client, title)
|
|
(card_type, instance, language_count), score, views = await asyncio.gather(
|
|
card_type_task, wikirank_task, pageviews_task
|
|
)
|
|
rarity = _score_to_rarity(score)
|
|
multiplier = RARITY_MULTIPLIER[rarity]
|
|
|
|
attack = int(language_count*1.5*multiplier**2)
|
|
defense = int(_pageviews_to_defense(views)*max(multiplier,(multiplier**2)/2))
|
|
|
|
return Card(
|
|
name=summary["title"],
|
|
created_at=datetime.now(),
|
|
image_link=summary.get("thumbnail", {}).get("source", ""),
|
|
card_rarity=rarity,
|
|
card_type=card_type,
|
|
wikidata_instance=instance,
|
|
text=text,
|
|
attack=attack,
|
|
defense=defense,
|
|
cost=min(12,max(1,int(cbrt(attack+defense)/1.5)))
|
|
)
|
|
|
|
async def _get_cards_async(size: int) -> list[Card]:
|
|
async with httpx.AsyncClient(follow_redirects=True) as client:
|
|
cards = await asyncio.gather(*[_get_card_async(client) for _ in range(size)])
|
|
return [c for c in cards if c is not None]
|
|
|
|
async def _get_specific_card_async(title: str) -> Card|None:
|
|
async with httpx.AsyncClient(follow_redirects=True) as client:
|
|
return await _get_card_async(client, title)
|
|
|
|
# Sync entrypoints
|
|
def generate_cards(size: int) -> list[Card]:
|
|
print(f"Generating {size} cards")
|
|
batches = [10 for _ in range(size//10)] + ([size%10] if size%10 != 0 else [])
|
|
n_batches = len(batches)
|
|
cards = []
|
|
for i in range(n_batches):
|
|
b = batches[i]
|
|
print(f"Generating batch of {b} cards (batch {i+1}/{n_batches})")
|
|
if i != 0:
|
|
sleep(5)
|
|
cards += asyncio.run(_get_cards_async(b))
|
|
|
|
return cards
|
|
|
|
def generate_card(title: str) -> Card|None:
|
|
return asyncio.run(_get_specific_card_async(title))
|
|
|
|
# for card in generate_cards(5):
|
|
# print(card)
|
|
|
|
# rarities = []
|
|
# from collections import Counter
|
|
# for card in generate_cards(1000):
|
|
# rarities.append(card.card_rarity)
|
|
# if card.card_rarity == CardRarity.legendary:
|
|
# print(card)
|
|
|
|
# print(Counter(rarities))
|
|
|
|
# for card in generate_cards(100):
|
|
# if card.card_type == CardType.other:
|
|
# print(card) |