From 3d2d9b7b65f0ae3cbe48ff6ac39411880b278032 Mon Sep 17 00:00:00 2001 From: Olivier Date: Wed, 13 Jan 2021 20:54:06 +0000 Subject: [PATCH] Allow using the eBay API Fixes #1. Need to set an eBay API key using EBAY_KEY env var. --- fleabay/api_extractor.py | 210 +++++++++++++++++++++++- fleabay/definitions.py | 23 ++- fleabay/extractor.py | 5 +- fleabay/templates/search_form.f.html.j2 | 5 + fleabay/utils.py | 15 ++ fleabay/webapp.py | 51 ++++-- 6 files changed, 294 insertions(+), 15 deletions(-) create mode 100644 fleabay/utils.py diff --git a/fleabay/api_extractor.py b/fleabay/api_extractor.py index 0e72162..e5edc0e 100644 --- a/fleabay/api_extractor.py +++ b/fleabay/api_extractor.py @@ -1,5 +1,211 @@ +import itertools +import os +import pprint +from typing import List, Dict + +from immutabledict import immutabledict +from requests import Session + +from fleabay.definitions import EbayItemSummary, EbayItemDetails, EbayMultiItemModelAxis +from fleabay.utils import chunker, decimal_price_to_pence + class EbayApiExtractor: - def __init__(self, api_key): - pass + def __init__(self, api_key: str): + self.api_key = api_key + self.session = Session() + def search(self, term: str, items_per_page: int = 200, remote_lowest_first: bool = False) -> List[EbayItemSummary]: + url = "https://svcs.ebay.com/services/search/FindingService/v1" + params = { + "SECURITY-APPNAME": self.api_key, + "OPERATION-NAME": "findItemsByKeywords", + "SERVICE-VERSION": "1.0.0", + "RESPONSE-DATA-FORMAT": "JSON", + "REST-PAYLOAD": "", + "keywords": term, + "paginationInput.entriesPerPage": items_per_page, + "GLOBAL-ID": "EBAY-GB", + "siteid": 3, + "buyerPostalCode": "M11 2GX" + } + if remote_lowest_first: + params["sortOrder"] = "PricePlusShippingLowest" + + resp = self.session.get(url, params=params).json() + resp_k = resp["findItemsByKeywordsResponse"][0] + if resp_k["ack"][0] != "Success": + raise RuntimeError("search by API not successful.") + + items = [] + + found_item_ids = dict() + + if resp_k["searchResult"][0]["@count"] == "0": + return [] + + try: + item_results = resp_k["searchResult"][0]["item"] + except KeyError as e: + raise RuntimeError("%r" % resp_k["searchResult"][0].items()) from e + + for result in item_results: + item_id = result["itemId"][0] + + if item_id in found_item_ids: + continue + + if item_id == "254817515397": + pprint.pprint(result) + + is_buy_it_now = result["listingInfo"][0]["listingType"][0] != "Auction" + + postage_price = result["shippingInfo"][0]["shippingServiceCost"][0] + postage_pence = -42 + if postage_price["@currencyId"] == "GBP": + postage_pence = int(100 * float(postage_price["__value__"])) + + if result["shippingInfo"][0]["shippingType"][0] == "FreePickup": + postage_pence = None + + item_price = result["sellingStatus"][0]["currentPrice"][0] + item_pence = -42 + + bid_count = result["sellingStatus"][0].get("bidCount") + if bid_count is not None: + bid_count = bid_count[0] + + if item_price["@currencyId"] == "GBP": + item_pence = int(100 * float(item_price["__value__"])) + + view_url = f"https://www.ebay.co.uk/itm/{item_id}" + + try: + condition = result["condition"][0]["conditionDisplayName"][0] + except KeyError: + condition = "???" + + if item_id == "154280943581": + pprint.pprint(result) + + item = EbayItemSummary( + item_id, view_url, result["title"][0], condition, + item_pence, -42, None if is_buy_it_now else bid_count, result["country"][0], postage_pence, result["galleryURL"][0] + ) + + found_item_ids[item_id] = item + + items.append(item) + + for item_id, details in self.details_batch(list(found_item_ids.keys())).items(): + summary = found_item_ids[item_id] + summary.extras["details"] = details + summary.min_price = int(details.extra["min_price"]) + summary.max_price = int(details.extra["max_price"]) + + return items + + @staticmethod + def details(summary: EbayItemSummary) -> EbayItemDetails: + return summary.extras["details"] + + def details_batch(self, ids: List[str]) -> Dict[str, EbayItemDetails]: + # eBay can do up to 20 items at once. + result = dict() + for id_chunk in chunker(ids, 20): + url = "https://open.api.ebay.com/shopping" + params = { + "ItemID": ",".join(id_chunk), + "callname": "GetMultipleItems", + "IncludeSelector": "Details,Variations", + "appid": self.api_key, + "siteid": 3, + "version": "967", + "responseencoding": "JSON" + } + response = self.session.get(url, params=params) + # print(response.status_code) + + resp = response.json() + + for item_dict in resp["Item"]: + item_id = item_dict["ItemID"] + seller_dict = item_dict["Seller"] + seller = { + "name": seller_dict["UserID"], + "feedback": f"{seller_dict['PositiveFeedbackPercent']}% ({seller_dict['FeedbackScore']})" + } + + extras = { + "variant_extras": dict() + } + + base_pence = -42 + if item_dict["CurrentPrice"]["CurrencyID"] == "GBP": + base_pence = decimal_price_to_pence("%.2f" % item_dict["CurrentPrice"]["Value"]) + + min_pence = base_pence + max_pence = base_pence + + variants_dict = dict() + axes: Dict[str, EbayMultiItemModelAxis] = dict() + axes_inverse: Dict[str, Dict[str, int]] = dict() + + variations = item_dict.get("Variations") + if variations is not None: + min_pence = 1e9 + max_pence = -42 + + for axis in variations["VariationSpecificsSet"]["NameValueList"]: + axes_inverse[axis["Name"]] = { + val: i for i, val in enumerate(axis["Value"]) + } + axes[axis["Name"]] = EbayMultiItemModelAxis(axis["Name"], dict(enumerate(axis["Value"]))) + + for variation in variations["Variation"]: + quantity = variation["Quantity"] - variation["SellingStatus"].get("QuantitySold", 0) + + if quantity <= 0: + continue + + price = -42 + if variation["StartPrice"]["CurrencyID"] == "GBP": + price = decimal_price_to_pence("%.2f" % variation["StartPrice"]["Value"]) + + min_pence = min(min_pence, price) + max_pence = max(max_pence, price) + + sold = variation["SellingStatus"]["QuantitySold"] + + variant_immdict = immutabledict({ + pair["Name"]: axes_inverse[pair["Name"]][pair["Value"][0]] + for pair in variation["VariationSpecifics"]["NameValueList"] + }) + + extras["variant_extras"][variant_immdict] = { + "quantity": quantity, + "sold": sold + } + + assert isinstance(price, int) + variants_dict[variant_immdict] = price + + if item_id == "223749865106": + pprint.pprint(variation) + print("vi", "%r" % variant_immdict) + print("vp", "%r" % price) + print("qt", "%r" % quantity) + + extras["min_price"] = int(min_pence) + extras["max_price"] = int(max_pence) + + result[item_id] = EbayItemDetails( + seller_dict["UserID"], + int(seller_dict["FeedbackScore"]), + seller_dict["PositiveFeedbackPercent"], + axes, + variants_dict, + extras + ) + + return result diff --git a/fleabay/definitions.py b/fleabay/definitions.py index 3d5715f..df391d8 100644 --- a/fleabay/definitions.py +++ b/fleabay/definitions.py @@ -1,4 +1,4 @@ -from typing import Optional, Dict +from typing import Optional, Dict, Any import attr from immutabledict import immutabledict @@ -6,6 +6,8 @@ from immutabledict import immutabledict @attr.s(auto_attribs=True) class EbayItemSummary: + # arbitrary ID + id: str href: str name: str condition: str @@ -18,6 +20,9 @@ class EbayItemSummary: postage: Optional[int] image_src: str + # extra data about the item. + extras: Dict[str, Any] = attr.attrib(factory=dict) + @attr.s(auto_attribs=True) class EbayMultiItemModelAxis: @@ -35,3 +40,19 @@ class EbayItemDetails: axes: Dict[str, EbayMultiItemModelAxis] # variants: Dict[immutabledict[str, int]] variants: Dict[immutabledict, int] + + extra: Dict[str, Any] = attr.attrib(factory=dict) + + +codes_to_symbols = { + "GBP": "£", + "EUR": "€", + "USD": "$", + "JPY": "¥" +} + + +@attr.s(auto_attribs=True) +class Price: + currency_code: str + amount: int diff --git a/fleabay/extractor.py b/fleabay/extractor.py index 995619d..0e78b5c 100644 --- a/fleabay/extractor.py +++ b/fleabay/extractor.py @@ -107,6 +107,7 @@ class EbayExtractor: results.append( EbayItemSummary( + link_href, link_href, title, condition, @@ -121,8 +122,8 @@ class EbayExtractor: return results - def details(self, url: str) -> EbayItemDetails: - page = self.session.get(url) + def details(self, summary: EbayItemSummary) -> EbayItemDetails: + page = self.session.get(summary.href) line: bytes for line in page.iter_lines(): opening = line.find(b"$rwidgets([") diff --git a/fleabay/templates/search_form.f.html.j2 b/fleabay/templates/search_form.f.html.j2 index 6c2ec2a..a2fbfec 100644 --- a/fleabay/templates/search_form.f.html.j2 +++ b/fleabay/templates/search_form.f.html.j2 @@ -8,6 +8,11 @@ +
+ {{ search_form.extractor.label }} + {{ search_form.extractor(class="form-control") }} +
+
{{ search_form.remote_lowest_first }} {{ search_form.remote_lowest_first.label() }}
diff --git a/fleabay/utils.py b/fleabay/utils.py new file mode 100644 index 0000000..135c395 --- /dev/null +++ b/fleabay/utils.py @@ -0,0 +1,15 @@ +import re + + +def decimal_price_to_pence(price: str) -> int: + pmatch = re.search(r"£?([0-9]+)\.([0-9]+)", price) + if pmatch is None: + raise ValueError( + "Unable to extract price from " + price + ) + return int(pmatch.group(1)) * 100 + int(pmatch.group(2)) + + +# https://stackoverflow.com/a/434328 CC BY-SA 4.0 +def chunker(seq, size): + return (seq[pos:pos + size] for pos in range(0, len(seq), size)) diff --git a/fleabay/webapp.py b/fleabay/webapp.py index 9c5a207..5c06142 100644 --- a/fleabay/webapp.py +++ b/fleabay/webapp.py @@ -1,12 +1,15 @@ import asyncio +import os from concurrent.futures.thread import ThreadPoolExecutor -from typing import Dict +from pprint import pprint +from typing import Dict, Any, Tuple import quart from quart import Quart, render_template, request -from wtforms_async import Form, StringField, BooleanField +from wtforms_async import Form, StringField, BooleanField, SelectField from wtforms_async.widgets import CheckboxInput +from fleabay.api_extractor import EbayApiExtractor from fleabay.definitions import EbayItemSummary, EbayItemDetails from fleabay.extractor import EbayExtractor @@ -15,7 +18,32 @@ app = Quart(__name__) pool = ThreadPoolExecutor(8) +def make_extractors() -> Tuple[Dict[str, Any], Dict[str, str]]: + available_extractors = dict() + extractor_names = dict() + + ebay_key = os.environ.get("EBAY_KEY") + if ebay_key: + available_extractors["eBay_api"] = EbayApiExtractor(ebay_key) + extractor_names["eBay_api"] = "eBay (by API — best)" + + available_extractors["eBay_scrape"] = EbayExtractor() + extractor_names["eBay_scrape"] = "eBay (by scraper — slow, error-prone)" + + zapiex_key = os.environ.get("ZAPIEX_KEY") + if zapiex_key: + available_extractors["AE_zapiex"] = None + extractor_names["AE_zapiex"] = "AliExpress (by Zapiex API — limited)" + + return available_extractors, extractor_names + + +# we need only initialise the extractors once. +available_extractors, extractor_names = make_extractors() + + class SearchForm(Form): + extractor = SelectField("Site/Extractor", choices=list(extractor_names.items())) search_term = StringField("Search Term") include_local_collection = BooleanField("Include local collection") remote_lowest_first = BooleanField("Ask remote for 'Lowest Price+P&P First'") @@ -33,13 +61,13 @@ async def search(): if not await search_form.validate(): return await render_template("index.html.j2", search_form=SearchForm()) - ex = EbayExtractor() + ex = available_extractors[search_form.extractor.data] search_term = search_form.search_term.data loop = asyncio.get_running_loop() - results = await loop.run_in_executor(pool, ex.search, search_term, 50) + results = await loop.run_in_executor(pool, ex.search, search_term, 100, search_form.remote_lowest_first.data) if not search_form.include_local_collection.data: results = [r for r in results if r.postage is not None] @@ -47,9 +75,9 @@ async def search(): item_details_fut = dict() for result in results: - if result.min_price != result.max_price: - fut = loop.run_in_executor(pool, ex.details, result.href) - item_details_fut[fut] = result.href + if not isinstance(ex, EbayExtractor) or result.min_price != result.max_price: + fut = loop.run_in_executor(pool, ex.details, result) + item_details_fut[fut] = result.id futs = tuple(item_details_fut.keys()) detail_results = await asyncio.gather(*futs, return_exceptions=True) @@ -70,7 +98,7 @@ async def search(): # want to find the most insightful axis to display over # Insightful suggests that the individual values in the axis have a tight range axes_ranges = dict() - for href, details in item_details.items(): + for item_id, details in item_details.items(): best_axis = None best_axis_tightness = 1.0e9 for axis in details.axes.values(): @@ -96,15 +124,18 @@ async def search(): for value, value_name in best_axis.values.items(): min_vp = 1e9 max_vp = 0 + found_any = False for variant, price in details.variants.items(): if variant[best_axis.code_name] != value: continue max_vp = max(max_vp, price) min_vp = min(min_vp, price) + found_any = True - axis_value_ranges[value_name] = (min_vp, max_vp) + if found_any: + axis_value_ranges[value_name] = (int(min_vp), int(max_vp)) - axes_ranges[href] = (best_axis.code_name, axis_value_ranges) + axes_ranges[item_id] = (best_axis.code_name, axis_value_ranges) return await render_template( "search.html.j2",