Python

2021-11-24 (Last Modified: 2023-01-31)

Page content

Python Snippets

RealPython Best Practices: https://realpython.com/tutorials/best-practices/

Convert CSV to JSON

cat LARGEFILE.csv |python3 -c 'import csv, json, sys; print(json.dumps([dict(r) for r in csv.DictReader(sys.stdin)]))' > LARGEFILE.json

### Show System Path

python3.10 -c "import sys; print('\n'.join(sys.path))"

Zfill

Padding Zero

>>> for i in range(1,10):
...     str(i).zfill(4)
'0001'
'0002'
'0003'
...

Different Padding

#!/usr/bin/env python3
for i in range(1,99):
  print( str(i).zfill(4) +" "+ str(i).rjust(4, '-') +" "+ str(i).ljust(4, '-') +" "+ str(i).center(4, '-') )

0001 ---1 1--- -1--
0002 ---2 2--- -2--
0003 ---3 3--- -3--
...

Padding with Function

#!/usr/bin/env python3

len=8
pad='-'

def my_print(i,len=4,pad='_'):
  print( str(i).zfill(len) +" "+ str(i).rjust(len, pad) +" "+ str(i).center(len, pad) +" "+ str(i).ljust(len, pad) )

for i in range(8,12):
  my_print(i,len,pad)

for i in range(98,102):
  my_print(i,len,pad)

for i in range(998,1002):
  my_print(i,len,pad)

00000008 -------8 ---8---- 8-------
00000009 -------9 ---9---- 9-------
00000010 ------10 ---10--- 10------
00000011 ------11 ---11--- 11------
00000098 ------98 ---98--- 98------
00000099 ------99 ---99--- 99------
00000100 -----100 --100--- 100-----
00000101 -----101 --101--- 101-----
00000998 -----998 --998--- 998-----
00000999 -----999 --999--- 999-----
00001000 ----1000 --1000-- 1000----
00001001 ----1001 --1001-- 1001----

DNS Lookup

python3 -c 'import socket; print(socket.gethostbyname("www.stoege.net"))'
116.203.179.238

Reverse Lookup

python3 -c 'import socket; print(socket.gethostbyaddr("116.203.179.238"))'
('static.238.179.203.116.clients.your-server.de', [], ['116.203.179.238'])

import dns.reversename
n = dns.reversename.from_address("116.203.179.238")
print(n)
print(dns.reversename.to_address(n))

238.179.203.116.in-addr.arpa.
116.203.179.238

Array with IP Adresses

import ipaddress
ip_pool = ipaddress.IPv4Network("10.1.2.0/27")
addrs = [str(p.network_address) for p in ip_pool.subnets(new_prefix=32)]
addrs
['10.1.2.0', '10.1.2.1', '10.1.2.2', '10.1.2.3', '10.1.2.4', '10.1.2.5', '10.1.2.6', '10.1.2.7', '10.1.2.8', '10.1.2.9', '10.1.2.10', '10.1.2.11', '10.1.2.12', '10.1.2.13', '10.1.2.14', '10.1.2.15', '10.1.2.16', '10.1.2.17', '10.1.2.18', '10.1.2.19', '10.1.2.20', '10.1.2.21', '10.1.2.22', '10.1.2.23', '10.1.2.24', '10.1.2.25', '10.1.2.26', '10.1.2.27', '10.1.2.28', '10.1.2.29', '10.1.2.30', '10.1.2.31']

OneLiner

python -c 'print ("A"*5)'
AAAAA

DataTypes

Lists: Array of Values
foo = [2, 4, 9, 1, 7]

Tuples: Tuples can be looked at as unmodifiable lists.
tup = (1, 2, 3, 5, 8)

Dicts:  Maps are called dictionaries in Python
shapeCorners = { "Triangle": 3, "Rectangle": 4, "Pentagon": 5, }

Sets: Sets are collections that contain unique items - there cannot be any duplicate
foods = { "coconuts", "spam" }

Reverse and Forward Name Check

is your IP - Host - IP mapping consistent … ? No, it is not … !

import socket

# Loop over 192.168.108.0/23 (-> 192.168.108.1 - 192.168.109.254)
for c in range(108, 110):
    for d in range(1, 255):
        ip = '192.168.'+str(c)+'.'+str(d)
        try:
            host = socket.gethostbyaddr(ip)[0]
        except:
            host = '*** NOT SET ***'

        try:
            ip_check = socket.gethostbyname(host)
        except:
            ip_check = '*** NOT SET ***'

        if ip == ip_check:
            print("OK  ", "ip:", ip.ljust(18), "host:", host.ljust(20), "ip:", ip_check.ljust(18))
        else:
            print("NOK ", "ip:", ip.ljust(18), "host:", host.ljust(20), "ip:", ip_check.ljust(18))

Example

...
NOK  ip: 192.168.108.16     host: *** NOT SET ***      ip: *** NOT SET ***
NOK  ip: 192.168.108.17     host: *** NOT SET ***      ip: *** NOT SET ***
OK   ip: 192.168.108.18     host: cookie-old           ip: 192.168.108.18
OK   ip: 192.168.108.19     host: cookie               ip: 192.168.108.19
...

Import CSV, add FQDN

Dump the ARP Table to a CSV File, add FQDN with Python. A little Example with CSV and Pandas

import pandas as pd

"""
GET ARP Table -> hosts.csv
OBSD:  arp -an |awk '/:/ {printf "%s,%s\n", $2, $1 }' |sort > hosts.csv
macOS: arp -an |awk -F'[ ()]' '/:/ {printf "%s,%s\n",$6,$3}' |sort > hosts.csv
"""

file = 'hosts.csv'

def get_fqdn(ip_address):
    import socket
    try:
        result = list(socket.gethostbyaddr(ip_address))[0]
    except:
        result = "NO-FQDN"
    return result

# Main
df = pd.read_csv(file)
df["fqdn"] = df["ip"].map(lambda ip: get_fqdn(ip))
df.sort_values(["fqdn"], axis=0, ascending=[True], inplace=True)
df.to_csv(file, index=False)

hosts.csv (before)

mac,ip
00:0d:b9:aa:bb:cc,10.10.10.1
00:0d:b9:aa:bb:dd,10.10.10.2
00:00:00:00:00:01,192.168.1.1
00:00:00:00:00:02,192.168.1.2
00:00:00:00:00:03,192.168.1.3

hosts.csv (after)

mac,ip,fqdn
00:0d:b9:aa:bb:cc,10.10.10.1,NO-FQDN
00:0d:b9:aa:bb:dd,10.10.10.2,NO-FQDN
00:00:00:00:00:01,192.168.1.1,host1.home
00:00:00:00:00:02,192.168.1.2,host2.home
00:00:00:00:00:03,192.168.1.3,host3.home

Dict/Json

 # Create Dict
 json_dict = {}
 json_dict['tweets'] = []
 json_dict['tweets'].append({
            'id': 'example',
            'date' : 'example' ,
            'tweet-text' : 'example',
            'tags' : 'example'
        })

  # Convert and write to json
  with open('tweets.json', 'w') as output:
      json.dump(json_dict, output)

Listen

l = [42,98,77]
l.append(103)
l
[42, 98, 77, 103]

x = l.pop()
x
103

l.pop()
l
77

# Append
l = [42,98,77]
l2 = [8,69]
l.append(l2)
l
[42, 98, 77, [8, 69]]

# Extend

l = [42,98,77]
l2 = [8,69]
l.extend(l2)
l
[42, 98, 77, 8, 69]

>>> l = [42,98,77]; l.append("Hallo"); l
[42, 98, 77, 'Hallo']

>>> l = [42,98,77]; l.extend("Hallo"); l
[42, 98, 77, 'H', 'a', 'l', 'l', 'o']

>>> L = [3,4]; L = L + [42]; L
[3, 4, 42]

>>> L = [3,4]; L += [42]; L
[3, 4, 42]

Farben = ["rot", "grün", "blau", "grün", "gelb"]
Farben.remove("grün")
Farben
['rot', 'blau', 'grün', 'gelb']

Farben = ["red", "green", "blue", "green", "yellow"]
Farben.index("green")
1

Dictionary

>>> woerter = {"house" : "Haus", "cat":"Katze", "black":"schwarz"}
>>> woerter["house"]
'Haus'

>>> len(woerter)
3

>>> del woerter["house"]
>>> len(woerter)
2

>>> if "cat" in woerter: print woerter["cat"]
Katze

# Copy
>>> w = woerter.copy()
>>> woerter["cat"]="chat"
>>> print woerter
{'house': 'Haus', 'cat': 'chat'}

# Clear
>>> w.clear()
>>> print w
{}

# Extend
>>> w={"house":"Haus","cat":"Katze","red":"rot"}
>>> w1 = {"red":"rouge","blau":"bleu"}
>>> w.update(w1)
>>> print w
{'house': 'Haus', 'blau': 'bleu', 'red': 'rouge', 'cat': 'Katze'}

# Iteration
for key in d:
	print key

for key in d.iterkeys():
	print key

for val in d.itervalues():
	print val

for key in d:
	print d[key]

# Dictionaries in Listen
>>> w={"house":"Haus","cat":"Katze","red":"rot"}
>>> w.items()
[('house', 'Haus'), ('red', 'rot'), ('cat', 'Katze')]
>>> w.keys()
['house', 'red', 'cat']
>>> w.values()
['Haus', 'rot', 'Katze']

# Listen on Dictionares
>>> gerichte = ["Pizza", "Sauerkraut", "Paella", "Hamburger"]
>>> laender = ["Italien","Deutschland","Spanien","USA"]
>>> land_gericht = zip(laender,gerichte)
>>> print land_gericht
[('Italien', 'Pizza'), ('Deutschland', 'Sauerkraut'), ('Spanien', 'Paella'), ('USA', 'Hamburger')]

API

Running an API / Talking to API

main.py

from typing import Optional

from fastapi import FastAPI
from pydantic import BaseModel


class Item(BaseModel):
    name: str
    description: Optional[str] = None
    price: float
    tax: Optional[float] = None


app = FastAPI()


@app.post("/items/")
async def create_item(item: Item):
    return item

Run Server

poetry run uvicorn main:app --host 0.0.0.0 --port 8000

Call from Client

# cat testcall.sh
curl -s -X 'POST' \
  'http://testbox:8000/items/' \
  -H 'accept: application/json' \
  -H 'Content-Type: application/json' \
  -d '{
        "name":"Gugus",
        "description":"Details uf Gugus ...",
        "price":10,
        "tax":0.25
      }' | jq '.'

# Response
./testcall.sh
{
  "name": "Gugus",
  "description": "Details uf Gugus ...",
  "price": 10,
  "tax": 0.25
}

Format Strings


i1='1.2.3.4'
i2='10.1.2.3'
i3='192.168.1.1'

# align right
>>> print("{:>20}{:>20}{:>20}".format(i1,i2,i3))
             1.2.3.4            10.1.2.3         192.168.1.1

# align left
>>> print("{:<20}{:<20}{:<20}".format(i1,i2,i3))
1.2.3.4             10.1.2.3            192.168.1.1

# align centered
>>> print("{:^20}{:^20}{:^20}".format(i1,i2,i3))
      1.2.3.4             10.1.2.3          192.168.1.1

>>> octets = i1.split('.')
>>> print("{:<20}{:<20}{:<20}{:<20}".format(*octets))
1                   2                   3                   4

Modules

my_module1.py

ip_addr = "8.8.8.8"
print(f"DNS Server IP: {ip_addr}")

my_module2.py

def dns_ip(dns="8.8.8.8"):
  print(f"DNS Server: {dns}")

main.py

import my_module1

import my_module2
my_module2.dns_ip()

from my_module2 import dns_ip
dns_ip()

SearchPath

’’ means current working directory

>>> import sys
>>> from pprint import pprint
>>> pprint(sys.path)
['',
 '/Library/Frameworks/Python.framework/Versions/3.7/lib/python37.zip',
 '/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7',
 '/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/lib-dynload',
 '/Users/username/Library/Python/3.7/lib/python/site-packages',
 '/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages']
>>>

Update SearchPath

env | grep PYTHON
export PYTHONWARNING=ignore::Warning
export PYTHONPATH=/bli/bla/blu

modules -> main

def main():
    print("Hello World!")

if __name__ == "__main__":
    main()

psutils - systeminfo

python3 -c "import psutil; print('MEM Usage:', psutil.virtual_memory().percent, '%')"

MEM Usage: 17.3 %

Python Path

show search path and extend with a own path

python3
import sys
from pprint import pprint

pprint(sys.path)

mypath='~/.local/mylibs'
if mypath not in sys.path:
  sys.path.append(mypath)

pprint(sys.path)

Annotating with simple data structures

new_price: list[int] = [14,902,898]
new_immutable_price: tuple[int,int,int] = (388,392,299)
new_price_dict: dict[str,int] = {
    "item_1":240,
    "item_2":490,
}

Pydantic

from pydantic import BaseModel

class Blog(BaseModel):
    title: str 
    is_active: bool

Blog(title="My First Blog",is_active=True)

Blog(title='Second One',is_active='yup!')

ValidationError: 1 validation error for Blog
is_active
  value could not be parsed to a boolean (type=type_error.bool)

Pydantic - Getting dynamic values at runtime

import time
from pydantic import BaseModel,Field
from datetime import datetime

class Blog(BaseModel):
    title: str 
    created_at: datetime = Field(default_factory=datetime.now)
    is_active: bool

print(Blog(title="Our First Blog",is_active=True))
time.sleep(3)
print(Blog(title="Our Second Blog",is_active=True))

#Output: 
#title='...' created_at=datetime.datetime(2022, 10, 7, 15, 57, 46, 257846) is_active=True
#title='...' created_at=datetime.datetime(2022, 10, 7, 15, 57, 49, 261350) is_active=True

Python Cert Path

python3 -c "import ssl; print(ssl.get_default_verify_paths())"
DefaultVerifyPaths(cafile='/opt/homebrew/etc/openssl@1.1/cert.pem', capath=None, openssl_cafile_env='SSL_CERT_FILE', openssl_cafile='/opt/homebrew/etc/openssl@1.1/cert.pem', openssl_capath_env='SSL_CERT_DIR', openssl_capath='/opt/homebrew/etc/openssl@1.1/certs')

get Cert

python3 -c "import ssl; print(ssl.get_server_certificate(('blog.stoege.net','443')))"

'-----BEGIN CERTIFICATE-----\nMIIGNTCCBR2gAwIBAgISA6As ...
..........................................................
... A9i9ba7tmT\n43xaxCG5M+hB\n-----END CERTIFICATE-----\n'

Tshoot

add pvlib on macos

https://www.logcg.com/en/archives/3548.html

poetry add pvlib
-> fatal error: 'hdf5.h' file not found

brew install hdf5
export CPATH="/usr/local/include/"
export HDF5_DIR="/usr/local/"
poetry add pvlib
-> works

Any Comments ?

sha256: 20d53b1f3967a001566579b569af8b68a50dca1dc69c5d2f9ce23ae2f90a130f