Allow using the eBay API
Fixes #1. Need to set an eBay API key using EBAY_KEY env var.
This commit is contained in:
parent
c0a726174e
commit
3d2d9b7b65
@ -1,5 +1,211 @@
|
||||
import itertools
|
||||
import os
|
||||
import pprint
|
||||
from typing import List, Dict
|
||||
|
||||
from immutabledict import immutabledict
|
||||
from requests import Session
|
||||
|
||||
from fleabay.definitions import EbayItemSummary, EbayItemDetails, EbayMultiItemModelAxis
|
||||
from fleabay.utils import chunker, decimal_price_to_pence
|
||||
|
||||
|
||||
class EbayApiExtractor:
|
||||
def __init__(self, api_key):
|
||||
pass
|
||||
def __init__(self, api_key: str):
|
||||
self.api_key = api_key
|
||||
self.session = Session()
|
||||
|
||||
def search(self, term: str, items_per_page: int = 200, remote_lowest_first: bool = False) -> List[EbayItemSummary]:
|
||||
url = "https://svcs.ebay.com/services/search/FindingService/v1"
|
||||
params = {
|
||||
"SECURITY-APPNAME": self.api_key,
|
||||
"OPERATION-NAME": "findItemsByKeywords",
|
||||
"SERVICE-VERSION": "1.0.0",
|
||||
"RESPONSE-DATA-FORMAT": "JSON",
|
||||
"REST-PAYLOAD": "",
|
||||
"keywords": term,
|
||||
"paginationInput.entriesPerPage": items_per_page,
|
||||
"GLOBAL-ID": "EBAY-GB",
|
||||
"siteid": 3,
|
||||
"buyerPostalCode": "M11 2GX"
|
||||
}
|
||||
if remote_lowest_first:
|
||||
params["sortOrder"] = "PricePlusShippingLowest"
|
||||
|
||||
resp = self.session.get(url, params=params).json()
|
||||
resp_k = resp["findItemsByKeywordsResponse"][0]
|
||||
if resp_k["ack"][0] != "Success":
|
||||
raise RuntimeError("search by API not successful.")
|
||||
|
||||
items = []
|
||||
|
||||
found_item_ids = dict()
|
||||
|
||||
if resp_k["searchResult"][0]["@count"] == "0":
|
||||
return []
|
||||
|
||||
try:
|
||||
item_results = resp_k["searchResult"][0]["item"]
|
||||
except KeyError as e:
|
||||
raise RuntimeError("%r" % resp_k["searchResult"][0].items()) from e
|
||||
|
||||
for result in item_results:
|
||||
item_id = result["itemId"][0]
|
||||
|
||||
if item_id in found_item_ids:
|
||||
continue
|
||||
|
||||
if item_id == "254817515397":
|
||||
pprint.pprint(result)
|
||||
|
||||
is_buy_it_now = result["listingInfo"][0]["listingType"][0] != "Auction"
|
||||
|
||||
postage_price = result["shippingInfo"][0]["shippingServiceCost"][0]
|
||||
postage_pence = -42
|
||||
if postage_price["@currencyId"] == "GBP":
|
||||
postage_pence = int(100 * float(postage_price["__value__"]))
|
||||
|
||||
if result["shippingInfo"][0]["shippingType"][0] == "FreePickup":
|
||||
postage_pence = None
|
||||
|
||||
item_price = result["sellingStatus"][0]["currentPrice"][0]
|
||||
item_pence = -42
|
||||
|
||||
bid_count = result["sellingStatus"][0].get("bidCount")
|
||||
if bid_count is not None:
|
||||
bid_count = bid_count[0]
|
||||
|
||||
if item_price["@currencyId"] == "GBP":
|
||||
item_pence = int(100 * float(item_price["__value__"]))
|
||||
|
||||
view_url = f"https://www.ebay.co.uk/itm/{item_id}"
|
||||
|
||||
try:
|
||||
condition = result["condition"][0]["conditionDisplayName"][0]
|
||||
except KeyError:
|
||||
condition = "???"
|
||||
|
||||
if item_id == "154280943581":
|
||||
pprint.pprint(result)
|
||||
|
||||
item = EbayItemSummary(
|
||||
item_id, view_url, result["title"][0], condition,
|
||||
item_pence, -42, None if is_buy_it_now else bid_count, result["country"][0], postage_pence, result["galleryURL"][0]
|
||||
)
|
||||
|
||||
found_item_ids[item_id] = item
|
||||
|
||||
items.append(item)
|
||||
|
||||
for item_id, details in self.details_batch(list(found_item_ids.keys())).items():
|
||||
summary = found_item_ids[item_id]
|
||||
summary.extras["details"] = details
|
||||
summary.min_price = int(details.extra["min_price"])
|
||||
summary.max_price = int(details.extra["max_price"])
|
||||
|
||||
return items
|
||||
|
||||
@staticmethod
|
||||
def details(summary: EbayItemSummary) -> EbayItemDetails:
|
||||
return summary.extras["details"]
|
||||
|
||||
def details_batch(self, ids: List[str]) -> Dict[str, EbayItemDetails]:
|
||||
# eBay can do up to 20 items at once.
|
||||
result = dict()
|
||||
for id_chunk in chunker(ids, 20):
|
||||
url = "https://open.api.ebay.com/shopping"
|
||||
params = {
|
||||
"ItemID": ",".join(id_chunk),
|
||||
"callname": "GetMultipleItems",
|
||||
"IncludeSelector": "Details,Variations",
|
||||
"appid": self.api_key,
|
||||
"siteid": 3,
|
||||
"version": "967",
|
||||
"responseencoding": "JSON"
|
||||
}
|
||||
response = self.session.get(url, params=params)
|
||||
# print(response.status_code)
|
||||
|
||||
resp = response.json()
|
||||
|
||||
for item_dict in resp["Item"]:
|
||||
item_id = item_dict["ItemID"]
|
||||
seller_dict = item_dict["Seller"]
|
||||
seller = {
|
||||
"name": seller_dict["UserID"],
|
||||
"feedback": f"{seller_dict['PositiveFeedbackPercent']}% ({seller_dict['FeedbackScore']})"
|
||||
}
|
||||
|
||||
extras = {
|
||||
"variant_extras": dict()
|
||||
}
|
||||
|
||||
base_pence = -42
|
||||
if item_dict["CurrentPrice"]["CurrencyID"] == "GBP":
|
||||
base_pence = decimal_price_to_pence("%.2f" % item_dict["CurrentPrice"]["Value"])
|
||||
|
||||
min_pence = base_pence
|
||||
max_pence = base_pence
|
||||
|
||||
variants_dict = dict()
|
||||
axes: Dict[str, EbayMultiItemModelAxis] = dict()
|
||||
axes_inverse: Dict[str, Dict[str, int]] = dict()
|
||||
|
||||
variations = item_dict.get("Variations")
|
||||
if variations is not None:
|
||||
min_pence = 1e9
|
||||
max_pence = -42
|
||||
|
||||
for axis in variations["VariationSpecificsSet"]["NameValueList"]:
|
||||
axes_inverse[axis["Name"]] = {
|
||||
val: i for i, val in enumerate(axis["Value"])
|
||||
}
|
||||
axes[axis["Name"]] = EbayMultiItemModelAxis(axis["Name"], dict(enumerate(axis["Value"])))
|
||||
|
||||
for variation in variations["Variation"]:
|
||||
quantity = variation["Quantity"] - variation["SellingStatus"].get("QuantitySold", 0)
|
||||
|
||||
if quantity <= 0:
|
||||
continue
|
||||
|
||||
price = -42
|
||||
if variation["StartPrice"]["CurrencyID"] == "GBP":
|
||||
price = decimal_price_to_pence("%.2f" % variation["StartPrice"]["Value"])
|
||||
|
||||
min_pence = min(min_pence, price)
|
||||
max_pence = max(max_pence, price)
|
||||
|
||||
sold = variation["SellingStatus"]["QuantitySold"]
|
||||
|
||||
variant_immdict = immutabledict({
|
||||
pair["Name"]: axes_inverse[pair["Name"]][pair["Value"][0]]
|
||||
for pair in variation["VariationSpecifics"]["NameValueList"]
|
||||
})
|
||||
|
||||
extras["variant_extras"][variant_immdict] = {
|
||||
"quantity": quantity,
|
||||
"sold": sold
|
||||
}
|
||||
|
||||
assert isinstance(price, int)
|
||||
variants_dict[variant_immdict] = price
|
||||
|
||||
if item_id == "223749865106":
|
||||
pprint.pprint(variation)
|
||||
print("vi", "%r" % variant_immdict)
|
||||
print("vp", "%r" % price)
|
||||
print("qt", "%r" % quantity)
|
||||
|
||||
extras["min_price"] = int(min_pence)
|
||||
extras["max_price"] = int(max_pence)
|
||||
|
||||
result[item_id] = EbayItemDetails(
|
||||
seller_dict["UserID"],
|
||||
int(seller_dict["FeedbackScore"]),
|
||||
seller_dict["PositiveFeedbackPercent"],
|
||||
axes,
|
||||
variants_dict,
|
||||
extras
|
||||
)
|
||||
|
||||
return result
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
from typing import Optional, Dict
|
||||
from typing import Optional, Dict, Any
|
||||
|
||||
import attr
|
||||
from immutabledict import immutabledict
|
||||
@ -6,6 +6,8 @@ from immutabledict import immutabledict
|
||||
|
||||
@attr.s(auto_attribs=True)
|
||||
class EbayItemSummary:
|
||||
# arbitrary ID
|
||||
id: str
|
||||
href: str
|
||||
name: str
|
||||
condition: str
|
||||
@ -18,6 +20,9 @@ class EbayItemSummary:
|
||||
postage: Optional[int]
|
||||
image_src: str
|
||||
|
||||
# extra data about the item.
|
||||
extras: Dict[str, Any] = attr.attrib(factory=dict)
|
||||
|
||||
|
||||
@attr.s(auto_attribs=True)
|
||||
class EbayMultiItemModelAxis:
|
||||
@ -35,3 +40,19 @@ class EbayItemDetails:
|
||||
axes: Dict[str, EbayMultiItemModelAxis]
|
||||
# variants: Dict[immutabledict[str, int]]
|
||||
variants: Dict[immutabledict, int]
|
||||
|
||||
extra: Dict[str, Any] = attr.attrib(factory=dict)
|
||||
|
||||
|
||||
codes_to_symbols = {
|
||||
"GBP": "£",
|
||||
"EUR": "€",
|
||||
"USD": "$",
|
||||
"JPY": "¥"
|
||||
}
|
||||
|
||||
|
||||
@attr.s(auto_attribs=True)
|
||||
class Price:
|
||||
currency_code: str
|
||||
amount: int
|
||||
|
||||
@ -107,6 +107,7 @@ class EbayExtractor:
|
||||
|
||||
results.append(
|
||||
EbayItemSummary(
|
||||
link_href,
|
||||
link_href,
|
||||
title,
|
||||
condition,
|
||||
@ -121,8 +122,8 @@ class EbayExtractor:
|
||||
|
||||
return results
|
||||
|
||||
def details(self, url: str) -> EbayItemDetails:
|
||||
page = self.session.get(url)
|
||||
def details(self, summary: EbayItemSummary) -> EbayItemDetails:
|
||||
page = self.session.get(summary.href)
|
||||
line: bytes
|
||||
for line in page.iter_lines():
|
||||
opening = line.find(b"$rwidgets([")
|
||||
|
||||
@ -8,6 +8,11 @@
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="form-group">
|
||||
{{ search_form.extractor.label }}
|
||||
{{ search_form.extractor(class="form-control") }}
|
||||
</div>
|
||||
|
||||
<div class="custom-checkbox d-inline-block mr-10">
|
||||
{{ search_form.remote_lowest_first }} {{ search_form.remote_lowest_first.label() }}
|
||||
</div>
|
||||
|
||||
15
fleabay/utils.py
Normal file
15
fleabay/utils.py
Normal file
@ -0,0 +1,15 @@
|
||||
import re
|
||||
|
||||
|
||||
def decimal_price_to_pence(price: str) -> int:
|
||||
pmatch = re.search(r"£?([0-9]+)\.([0-9]+)", price)
|
||||
if pmatch is None:
|
||||
raise ValueError(
|
||||
"Unable to extract price from " + price
|
||||
)
|
||||
return int(pmatch.group(1)) * 100 + int(pmatch.group(2))
|
||||
|
||||
|
||||
# https://stackoverflow.com/a/434328 CC BY-SA 4.0
|
||||
def chunker(seq, size):
|
||||
return (seq[pos:pos + size] for pos in range(0, len(seq), size))
|
||||
@ -1,12 +1,15 @@
|
||||
import asyncio
|
||||
import os
|
||||
from concurrent.futures.thread import ThreadPoolExecutor
|
||||
from typing import Dict
|
||||
from pprint import pprint
|
||||
from typing import Dict, Any, Tuple
|
||||
|
||||
import quart
|
||||
from quart import Quart, render_template, request
|
||||
from wtforms_async import Form, StringField, BooleanField
|
||||
from wtforms_async import Form, StringField, BooleanField, SelectField
|
||||
from wtforms_async.widgets import CheckboxInput
|
||||
|
||||
from fleabay.api_extractor import EbayApiExtractor
|
||||
from fleabay.definitions import EbayItemSummary, EbayItemDetails
|
||||
from fleabay.extractor import EbayExtractor
|
||||
|
||||
@ -15,7 +18,32 @@ app = Quart(__name__)
|
||||
pool = ThreadPoolExecutor(8)
|
||||
|
||||
|
||||
def make_extractors() -> Tuple[Dict[str, Any], Dict[str, str]]:
|
||||
available_extractors = dict()
|
||||
extractor_names = dict()
|
||||
|
||||
ebay_key = os.environ.get("EBAY_KEY")
|
||||
if ebay_key:
|
||||
available_extractors["eBay_api"] = EbayApiExtractor(ebay_key)
|
||||
extractor_names["eBay_api"] = "eBay (by API — best)"
|
||||
|
||||
available_extractors["eBay_scrape"] = EbayExtractor()
|
||||
extractor_names["eBay_scrape"] = "eBay (by scraper — slow, error-prone)"
|
||||
|
||||
zapiex_key = os.environ.get("ZAPIEX_KEY")
|
||||
if zapiex_key:
|
||||
available_extractors["AE_zapiex"] = None
|
||||
extractor_names["AE_zapiex"] = "AliExpress (by Zapiex API — limited)"
|
||||
|
||||
return available_extractors, extractor_names
|
||||
|
||||
|
||||
# we need only initialise the extractors once.
|
||||
available_extractors, extractor_names = make_extractors()
|
||||
|
||||
|
||||
class SearchForm(Form):
|
||||
extractor = SelectField("Site/Extractor", choices=list(extractor_names.items()))
|
||||
search_term = StringField("Search Term")
|
||||
include_local_collection = BooleanField("Include local collection")
|
||||
remote_lowest_first = BooleanField("Ask remote for 'Lowest Price+P&P First'")
|
||||
@ -33,13 +61,13 @@ async def search():
|
||||
if not await search_form.validate():
|
||||
return await render_template("index.html.j2", search_form=SearchForm())
|
||||
|
||||
ex = EbayExtractor()
|
||||
ex = available_extractors[search_form.extractor.data]
|
||||
|
||||
search_term = search_form.search_term.data
|
||||
|
||||
loop = asyncio.get_running_loop()
|
||||
|
||||
results = await loop.run_in_executor(pool, ex.search, search_term, 50)
|
||||
results = await loop.run_in_executor(pool, ex.search, search_term, 100, search_form.remote_lowest_first.data)
|
||||
|
||||
if not search_form.include_local_collection.data:
|
||||
results = [r for r in results if r.postage is not None]
|
||||
@ -47,9 +75,9 @@ async def search():
|
||||
item_details_fut = dict()
|
||||
|
||||
for result in results:
|
||||
if result.min_price != result.max_price:
|
||||
fut = loop.run_in_executor(pool, ex.details, result.href)
|
||||
item_details_fut[fut] = result.href
|
||||
if not isinstance(ex, EbayExtractor) or result.min_price != result.max_price:
|
||||
fut = loop.run_in_executor(pool, ex.details, result)
|
||||
item_details_fut[fut] = result.id
|
||||
|
||||
futs = tuple(item_details_fut.keys())
|
||||
detail_results = await asyncio.gather(*futs, return_exceptions=True)
|
||||
@ -70,7 +98,7 @@ async def search():
|
||||
# want to find the most insightful axis to display over
|
||||
# Insightful suggests that the individual values in the axis have a tight range
|
||||
axes_ranges = dict()
|
||||
for href, details in item_details.items():
|
||||
for item_id, details in item_details.items():
|
||||
best_axis = None
|
||||
best_axis_tightness = 1.0e9
|
||||
for axis in details.axes.values():
|
||||
@ -96,15 +124,18 @@ async def search():
|
||||
for value, value_name in best_axis.values.items():
|
||||
min_vp = 1e9
|
||||
max_vp = 0
|
||||
found_any = False
|
||||
for variant, price in details.variants.items():
|
||||
if variant[best_axis.code_name] != value:
|
||||
continue
|
||||
max_vp = max(max_vp, price)
|
||||
min_vp = min(min_vp, price)
|
||||
found_any = True
|
||||
|
||||
axis_value_ranges[value_name] = (min_vp, max_vp)
|
||||
if found_any:
|
||||
axis_value_ranges[value_name] = (int(min_vp), int(max_vp))
|
||||
|
||||
axes_ranges[href] = (best_axis.code_name, axis_value_ranges)
|
||||
axes_ranges[item_id] = (best_axis.code_name, axis_value_ranges)
|
||||
|
||||
return await render_template(
|
||||
"search.html.j2",
|
||||
|
||||
Loading…
Reference in New Issue
Block a user