155 lines
4.3 KiB
Python
Executable File
155 lines
4.3 KiB
Python
Executable File
import os
|
|
|
|
import boto3
|
|
import pandas as pd
|
|
from botocore.exceptions import NoCredentialsError, PartialCredentialsError
|
|
from dotenv import load_dotenv
|
|
from fetch import scrape_plato
|
|
from utils import get
|
|
|
|
|
|
def update_database(articles_df=None, database_file="/home/user/plato.parquet"):
|
|
if os.path.exists(database_file):
|
|
database_df = pd.read_parquet(database_file)
|
|
else:
|
|
database_df = None
|
|
|
|
if articles_df is None:
|
|
new_df = None if database_df is None else database_df.head(0)
|
|
else:
|
|
if database_df is None:
|
|
articles_df.to_parquet(database_file)
|
|
return articles_df, articles_df
|
|
|
|
compare = ["ean", "_price"]
|
|
check_df = pd.merge(
|
|
database_df[compare], articles_df[compare], how="right", indicator=True
|
|
)
|
|
new_df = (
|
|
check_df[check_df["_merge"] == "right_only"]
|
|
.drop(columns="_merge")
|
|
.merge(articles_df)
|
|
)
|
|
database_df = (
|
|
pd.concat([database_df, new_df])
|
|
.sort_values("_date")
|
|
.groupby("ean")
|
|
.last()
|
|
.reset_index()
|
|
)
|
|
database_df.to_parquet(database_file)
|
|
|
|
return database_df, new_df
|
|
|
|
|
|
def send_email(lines):
|
|
# Define the email parameters
|
|
SENDER = "mail@veenboer.xyz"
|
|
RECIPIENT = "rik.veenboer@gmail.com"
|
|
SUBJECT = "Aanbieding op plato!"
|
|
|
|
# The email body for recipients with non-HTML email clients
|
|
BODY_TEXT = ""
|
|
|
|
# The HTML body of the email
|
|
tmp = "\n".join(lines)
|
|
BODY_HTML = f"""<html>
|
|
<head></head>
|
|
<body>
|
|
{tmp}
|
|
</html>
|
|
"""
|
|
|
|
# The character encoding for the email
|
|
CHARSET = "UTF-8"
|
|
|
|
# Try to send the email
|
|
try:
|
|
client = boto3.client(
|
|
"ses", region_name="eu-west-1"
|
|
) # Change the region as needed
|
|
|
|
# Provide the contents of the email
|
|
response = client.send_email(
|
|
Destination={
|
|
"ToAddresses": [
|
|
RECIPIENT,
|
|
],
|
|
},
|
|
Message={
|
|
"Body": {
|
|
"Html": {
|
|
"Charset": CHARSET,
|
|
"Data": BODY_HTML,
|
|
},
|
|
"Text": {
|
|
"Charset": CHARSET,
|
|
"Data": BODY_TEXT,
|
|
},
|
|
},
|
|
"Subject": {
|
|
"Charset": CHARSET,
|
|
"Data": SUBJECT,
|
|
},
|
|
},
|
|
Source=SENDER,
|
|
)
|
|
# Display an error if something goes wrong.
|
|
except NoCredentialsError:
|
|
print("Credentials not available")
|
|
except PartialCredentialsError:
|
|
print("Incomplete credentials provided")
|
|
except Exception as e:
|
|
print(f"Error: {e}")
|
|
else:
|
|
print("Email sent! Message ID:"),
|
|
print(response["MessageId"])
|
|
|
|
|
|
def main(dry=False):
|
|
load_dotenv("/opt/.env")
|
|
|
|
local_ip = get("http://ifconfig.me", False).text
|
|
get_ip = get("http://ifconfig.me").text
|
|
print(f"Local IP = {local_ip}")
|
|
print(f"Request IP = {get_ip}")
|
|
assert local_ip != get_ip
|
|
|
|
artists = open("/home/user/artists.txt").read().strip().splitlines()
|
|
print(f"Number of known artists = {len(artists)}")
|
|
|
|
if dry:
|
|
articles_df = None
|
|
else:
|
|
articles_df = scrape_plato(get=get)
|
|
database_df, new_df = update_database(articles_df)
|
|
|
|
if dry:
|
|
new_df = database_df.sample(20)
|
|
|
|
print(f"Database size = {len(database_df)}")
|
|
print(f"New = {len(new_df)}")
|
|
|
|
# new_df = new_df[new_df['_artist'].isin(artists)].query('_price <= 25')
|
|
new_df = new_df.query('_price <= 25 and ean != ""')
|
|
print(f"Interesting = {len(new_df)}")
|
|
|
|
if new_df is not None and len(new_df):
|
|
message = []
|
|
for _, row in new_df.head(10).iterrows():
|
|
message.append(
|
|
f'<a href="https://www.platomania.nl{row.url}"><h1>NEW</h1></a>'
|
|
)
|
|
message.append("<ul>")
|
|
message.append(f"<li>[artist] {row.artist}</li>")
|
|
message.append(f"<li>[title] {row.title}</li>")
|
|
message.append(f"<li>[price] {row.price}</li>")
|
|
message.append(f"<li>[release] {row.release_date}</li>")
|
|
message.append("</ul>")
|
|
send_email(message)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
cwd = os.path.dirname(__file__)
|
|
main(dry=False)
|