python - manual | D E W = 露の世

from: Python Deep Dive Course - Accompanying Materials

from: Indently

# IO

# file open

open 返回的实际上是一个生成器对象。

# 读取控制台或者文件内容
import fileinput

for line in fileinput.input():
    print(line)

with fileinput.FileInput(files=("a.txt", "b.txt"), mode="r") as input:
    for idx, line in enumerate(input):
        if input.isfirstline():
            # If you want to look at the name of the file currently being read, we can use the fileinput.filename() method.
            # However, this will return None if no line has yet been read! So you can use this only after the first read.
            print(f"Reading file {input.filename()}...")
        print(line)


f = open(filename, 'r')
print(hasattr(file, "__next__"), hasattr(file, "__iter__")) # True True
print(next(f))
print(next(f))
f.close()

# file lines

# 计算行数
k = 0
with open('test.txt') as fid:
    for line in fid:
        k = k + 1
print(k)

# 或
print(len([line for line in open('test.txt') ]))

print(sum(1 for _ in open('test.txt')))

# file state

获取文件目录名，文件名
获取文件是否存在，是目录还是文件
获取文件大小
获取文件创建时间，上次访问时间，修改时间

import os, sys
import time
import datetime

print(__file__, sys.argv[0]) # python test.py => test.py test.py
path = r'E:\study_python\test\fast02.py'
print(os.path.basename(path), os.path.dirname(path), os.path.split(path))
# fast02.py E:\study_python\test ('E:\\study_python\\test', 'fast02.py')

print(os.path.exists(path)
     ,os.path.isfile(path)
     ,os.path.isdir(path)
     ,os.path.getsize(path)
     ,os.path.getatime(path)   # retrieve the last access time
     ,datetime.datetime.fromtimestamp(os.path.getmtime(path)) # retrieve the last modification time
     ,os.path.getctime(path)) # retrieve the creation time 
# True True False 1740 1536125104.7735167 2018-09-05 16:59:29.086361 1536125104.7735167

# os.walk

remove node_modules

import os
import shutil

node_modules_dirs = []
node_modules = 'node_modules'

for root, dirs, files in os.walk(r'D:\study\node'):
    for dir in dirs:
        if dir == node_modules:
            path = os.path.join(root, dir)
            # exclude child module's node_modules
            # 除了最后一个 node_modules, 路径中不再包含其它 node_modules
            if node_modules not in path[0:-len(node_modules)]:
                node_modules_dirs.append(path)
                print(path)
print(len(node_modules_dirs), node_modules_dirs)

for dir in node_modules_dirs:
    shutil.rmtree(dir)
    print(f"Remove {dir} success!")

上述方法缺点是遍历目录文件过多时，性能较差，比如实际找到项目目录的 node_modules 时不需要再向下遍历安装的模块中不符合条件的 node_modules, 这时可以使用如下方式。

import os
import shutil

node_modules_dirs = []

def walk(dir):
    for subpath in os.listdir(dir):
        path = os.path.join(dir, subpath)

        if os.path.isdir(path):
            if path.endswith('node_modules'):
                node_modules_dirs.append(path)
            else:
                walk(path)

walk(r'D:\study\node')

print(len(node_modules_dirs), node_modules_dirs)

for dir in node_modules_dirs:
    shutil.rmtree(dir)
    print(f"Remove {dir} success!")

# StringIO

from io import StringIO

# f = StringIO('hello, world')
# print(f.readline())

f = StringIO()
f.write('hello, world') # f.write() 定位在文件末尾
print(f.readline())  # 读取不到数据
f.seek(0)
print(f.readline())  # hello, world

# shutil.move and os.rename

shutil means “shell utilities”.

# os.rename() requires to include the file name in both the source and destination arguments
os.rename('path/to/file.txt', 'path/to/new/directory/file.txt')
# while shutil.move() requires the new directory as destination or include the filename
shutil.move('path/to/file.txt', 'path/to/new/directory/')
shutil.move('path/to/file.txt', 'path/to/new/directory/file.txt')


# if file exsists, os.rename will raise file exists error
os.rename('path/to/file.txt', 'path/to/new/directory/file.txt')
# if file exsists, shutil.move will raise file exists error
shutil.move('path/to/file.txt', 'path/to/new/directory/')
# if file exsists, shutil.move will cover but no error
shutil.move('path/to/file.txt', 'path/to/new/directory/file.txt')

# if file exsists, shutil.copy will cover, no error
shutil.copy('path/to/file.txt', 'path/to/new/directory/')
shutil.copy('path/to/file.txt', 'path/to/new/directory/file.txt')

# shutil.copy2 copy file with metadata

Keep the file modification time and so on.

from pathlib import Path
import shutil
from tkinter import filedialog

src_path: str = filedialog.askopenfilename(title="Pick a file")
directory: str = filedialog.askdirectory(title="Pick a location")
dst_path: Path = Path(directory) / "copy.jpg"

print(shutil.copy2(src=src_path, dst=dst_path))

# print

Prints the values to a stream, or to sys.stdout by default.

def print(
    *values: object,
    sep: str | None = " ",
    end: str | None = "\n",
    file: SupportsWrite[str] | None = None,
    flush: Literal[False] = False
) -> None

sep: string inserted between values, default a space.
end: string appended after the last value, default a newline.
file: a file-like object (stream); defaults to the current sys.stdout.
flush: whether to forcibly flush the stream.

print('*', end='\n') : 默认输出换行

print('*', end='') : 输出不换行

print('*', '*', sep=' ') : 默认以空格分隔多个值

print('*', '*', sep='@') : 用 @ 分隔多个值

num_stars = int(input('How many stars do you want ? '))
for _ in range(num_stars):
     print('*',end='')
# 4
# ****
print(*("*" for i in range(num_stars)), sep="-", end="")
# 4
# *-*-*-*

print('*', '*', sep=' ')   # * *
print('*', '*', sep='@')   # *@*


foods: list[str] = ["Apples", "Bananas", "Oranges"]
print(*foods, sep=", ", end=".\n") # Apples, Bananas, Oranges.

# print %s, %r

%r(repr) unambiguously recreate the object it represents 可以重建它所代表的对象。

s = "world"
print("hello {!s}".format(s), ',', "hello %s" % s)
print("hello {!r}".format(s), ',', "hello %r" % s)
# hello world , hello world
# hello 'world' , hello 'world'

d = datetime.date.today()
print("today is {!s}".format(d), "today is %s" % d)
print("today is {!r}".format(d), "today is %r" % d)
# today is 2019-10-28 today is 2019-10-28
# today is datetime.date(2019, 10, 28) today is datetime.date(2019, 10, 28)

__repr__() 方法返回一个实例的代码表示形式，通常用来重新构造这个实例。内置的 repr () 函数返回这个字符串，跟我们使用交互式解释器显示的值是一样的。
__str__() 方法将实例转换为一个字符串，使用 str () 或 print () 函数会输出这个字符串。

class Pair:
    def __init__(self, x, y):
        self.x = x
        self.y = y

    def __repr__(self):
        # !r 格式化代码指明输出使用 __repr__() 来代替默认的 __str__()
        return 'Pair({0.x!r}, {0.y!r})'.format(self)

    def __str__(self):
        return '({0.x!s}, {0.y!s})'.format(self)

# >>> p = Pair(3, 4)
# >>> p
# Pair(3, 4) # __repr__() output
# >>> print(p)
# (3, 4) # __str__() output

# fstring

n: int = 100000000
print(f'{n:_}')
print(f'{n:,}')
print(f"{n:.2e}")
# 100_000_000
# 100,000,000
# 1.00e+08


var: str = 'var'
print(f'{var:>20}')
print(f'{var:<20}')
print(f'{var:^20}')
#                  var
# var
#         var


from datetime import datetime

now: datetime = datetime.now()
print(f'{now:%d.%m.%y (%H:%M:%S)}')
print(f'{now:%c}')
print(f'{now:%I%p}')
# 21.02.24 (17:49:03)
# Wed Feb 21 17:49:03 2024
# 05PM
now: datetime = datetime.now()
print(f"{now:%Y-%m-%d}")
date_spec: str = "%Y-%m-%d"
print(f"{now:{date_spec}}") 
# 2025-12-18
# 2025-12-18


n: float = 1234.5678
print(f'{n:.2f}')
print(f'{n:.0f}')
print(f'{n:_.3f}'
# 1234.57
# 1235
# 1_234.568


a: int = 5
b: int = 10
var: str = 'hello'
print(f'{a + b = }')
print(f'{bool(a) = }')
print(f'{var = }')
# a + b = 15
# bool(a) = True
# var = 'hello'
      

custom_folder: str = "Babb"
path: str = fr"\Desktop\STUDY\py-deep-dive\{custom_folder}"
print(path) # \Desktop\STUDY\py-deep-dive\Babb
      
      
banana: str = "🍌"
name: str = "Babb"
today: date = datetime.now().date()
print(f"{today!s} {name!s} says: {banana!s}")
print(f"{today!r} {name!r} says: {banana!r}")
print(f"{today!a} {name!a} says: {banana!a}")
# 2025-12-18 Babb says: 🍌
# datetime.date(2025, 12, 18) 'Babb' says: '🍌'
# datetime.date(2025, 12, 18) 'Babb' says: '\U0001f34c'

# logical lines

a = [
    1,  # item 1
    2,  # item 2
    3,  # item 3
]


a = {
    "key1": 1,  # item 1
    "key2": 2,  # item 2
    "key3": 3,  # item 3
}


def func(
    a,
    b,  # comment
    c,
):
    pass


func(
    10,  # comment
    20,
    30,
)


a = True
b = True
c = True

if a and b and c:
    pass

# if a \
#     and b \
#     and c:
#     pass


a = """this\n
\tis 
    a string
"""

print(a)

# Container

# string index

s = 'hello'
# for(int i=len(s) -1 ; i > 0; i--)
print(s[::-1])  # olleh

# palindrome
s = 'PHP'
print(s == s[::-1]) # True

# string interning

import sys
import time


a = "hello"
b = "hello"
print(a is b)  # True

a = "the quick brown fox"
b = "the quick brown fox"
print(a is b)  # True

a = "the quick brown fox" * 1000
b = "the quick brown fox" * 1000
print(a is b)  # False


def compare_using_equals(n):
    a = "the quick brown fox" * 1000
    b = "the quick brown fox" * 1000
    for i in range(n):
        if a == b:
            pass


def compare_using_interning(n):
    a = sys.intern("the quick brown fox" * 1000)
    b = sys.intern("the quick brown fox" * 1000)
    for i in range(n):
        if a is b:
            pass


start = time.perf_counter()
compare_using_equals(10000000)
end = time.perf_counter()
print("equality", end - start)  # equality 4.051721299998462

start = time.perf_counter()
compare_using_interning(10000000)
end = time.perf_counter()
print("interning", end - start)  # interning 0.13192070001969114

# + and +=

a = [1, 2]
c = a + [3, 4]
# c = a + (3, 4) # TypeError: can only concatenate list (not "tuple") to list

a += (5, 6)      # += 实际调用的是 extend，参数可以是任意序列类型，并且 += 是 inpalce 操作
a.extend(range(3))

a = [1, 2, 3]
print(id(a), a)	# 2551381626112 [1, 2, 3]
b = range(4, 8, 2)
a += b
print(id(a), a) # 2551381626112 [1, 2, 3, 4, 6]

# collections type check

collections 模块定义了很多跟容器和迭代器（序列、映射、集合等）有关的抽象基类。

The Iterable abstract class was removed from collections in python 3.10，You can use Iterable from collections.abc instead.

numbers 库定义了跟数字对象（整数、浮点数、有理数等）有关的基类。

io 库定义了很多跟 I/O 操作相关的基类。可以用这些抽象基类检查数据类型。

import collections

# Check if x is a sequence
if isinstance(x, collections.Sequence):
    pass

# Check if x is iterable
if isinstance(x, collections.Iterable):
    pass

# Check if x has a size
if isinstance(x, collections.Sized):
    pass

# Check if x is a mapping
if isinstance(x, collections.Mapping):
    pass

# iterable

展开嵌套序列

from collections import Iterable
# from collections.abc import Iterable # from python 3.10

def flatten(items, ignore_types=(str, bytes)):
    for x in items:
        if isinstance(x, Iterable) and not isinstance(x, ignore_types):
            yield from flatten(x)
        else:
            yield x

    items = [1, 2, [3, 4, [5, 6], 'abc'], b'abc']
for x in flatten(items):
    print(x)

# iter

iter 函数可以接受一个可选的 callable 对象和一个标记（结尾）值作为输入参数。当以这种方式使用的时候，它会创建一个迭代器，这个迭代器会不断调用 callable 对象直到返回值和标记值相等为止。

f = open('log2.txt', 'r')
for chunk in iter(lambda:f.read(10), ''):
    print(chunk)
f.close()
# hello
# worl
# d

# generator

# my_str = "hello"

# for i in my_str:
#     print(i)

# my_list = [1, 2, 3]

# for i in my_list:
#     print(i)

# print(hasattr(my_str, "__iter__"), hasattr(my_list, "__iter__"))

# it = iter(my_str)
# while True:
#     try:
#         print(next(it))
#     except StopIteration:
#         break


from contextlib import contextmanager
import tracemalloc


def process_line(line):
    pass


filepath = "large_data.csv"


@contextmanager
def trace_memory():
    tracemalloc.start()

    yield

    current, peak = tracemalloc.get_traced_memory()
    print(f"Current memory usage : {current / 1024**2} MB")
    print(f"Peak memory usage : {peak / 1024**2} MB")
    tracemalloc.stop()


with trace_memory():
    with open(filepath, "r") as f:
        lines = f.readlines()

        for line in lines:
            process_line(line)


class LineIterator:
    def __init__(self, filepath):
        self.file = open(filepath, "r")

    def __iter__(self):
        return self

    def __next__(self):
        line = self.file.readline()
        if line:
            return line
        else:
            self.file.close()
            raise StopIteration


def line_generator(filepath):
    with open(filepath, "r") as f:
        for line in f:
            yield line


with trace_memory():
    line_iter = LineIterator(filepath)
    for line in line_iter:
        process_line(line)

with trace_memory():
    line_gen = line_generator(filepath)
    for line in line_gen:
        process_line(line)


# def generator(n):
#     for i in range(n):
#         print("before yield")
#         yield i
#         print("after yield")


# gen = generator(3)
# print(next(gen))
# print(next(gen))
# print("-" * 10)
# for i in gen:
#     print(i)


# def fib_generator():
#     cur, next = 0, 1
#     while True:
#         yield cur
#         cur, next = next, cur + next


# fib_gen = fib_generator()
# for _ in range(10):
#     print(next(fib_gen))

import math


class FactIter:
    def __init__(self, n):
        self.n = n
        self.i = 0

    def __iter__(self):
        return self

    def __next__(self):
        if self.i < self.n:
            result = math.factorial(self.i)
            self.i += 1
            return result
        else:
            raise StopIteration


fact_iter = FactIter(5)
for i in fact_iter:
    print(i)
# 1
# 1
# 2
# 6
# 24


def fact():
    i = 0

    def inner():
        nonlocal i
        result = math.factorial(i)
        i += 1
        return result

    return inner


fact_iter = iter(fact(), math.factorial(5))
for i in fact_iter:
    print(i)
# 1
# 1
# 2
# 6
# 24


def factorials(n):
    for i in range(n):
        yield math.factorial(i)


fact_iter = factorials(5)
print("__next__" in dir(fact_iter), "__iter__" in dir(fact_iter))  # True True
for i in fact_iter:
    print(i)
# 1
# 1
# 2
# 6
# 24

from collections import namedtuple


Card = namedtuple("Card", "rank suit")


class CardDeck:
    SUITS = ("Spades", "Hearts", "Diamonds", "Clubs")
    RANKS = tuple(range(2, 11)) + tuple("JQKA")

    def __iter__(self):
        return CardDeck.card_gen()

    def __reversed__(self):
        return CardDeck.reversed_card_gen()

    @staticmethod
    def card_gen():
        for suit in CardDeck.SUITS:
            for rank in CardDeck.RANKS:
                yield Card(rank, suit)

    @staticmethod
    def reversed_card_gen():
        for suit in reversed(CardDeck.SUITS):
            for rank in reversed(CardDeck.RANKS):
                yield Card(rank, suit)


print(list(CardDeck())[::13])
# [
#     Card(rank=2, suit="Spades"),
#     Card(rank=2, suit="Hearts"),
#     Card(rank=2, suit="Diamonds"),
#     Card(rank=2, suit="Clubs"),
# ]

print(list(reversed(CardDeck()))[::13])
# [
#     Card(rank="A", suit="Clubs"),
#     Card(rank="A", suit="Diamonds"),
#     Card(rank="A", suit="Hearts"),
#     Card(rank="A", suit="Spades"),
# ]

def averager():
    total = 0
    count = 0
    running_average = None

    while True:
        value = yield running_average
        total += value
        count += 1
        running_average = total / count


def running_averages(iterable):
    average = averager()
    next(average)
    for value in iterable:
        running_average = average.send(value)
        print(running_average)


running_averages([1, 2, 3, 4])

# 1.0
# 1.5
# 2.0
# 2.5

def parse_file(f_name):
    print('opening file...')
    f = open(f_name, 'r')
    try:
        dialect = csv.Sniffer().sniff(f.read(2000))
        f.seek(0)
        next(f)  # skip header row
        reader = csv.reader(f, dialect=dialect)
        for row in reader:
            try:
                yield row
            except GeneratorExit:
                print('got a close...')
                return
    finally:
        print('cleaning up...')
        f.close() 
        
parser = parse_file('cars.csv')
for row in itertools.islice(parser, 5):
    print(row)
    
# opening file...
# ['STRING', 'DOUBLE', 'INT', 'DOUBLE', 'DOUBLE', 'DOUBLE', 'DOUBLE', 'INT', 'CAT']
# ['Chevrolet Chevelle Malibu', '18.0', '8', '307.0', '130.0', '3504.', '12.0', '70', 'US']
# ['Buick Skylark 320', '15.0', '8', '350.0', '165.0', '3693.', '11.5', '70', 'US']
# ['Plymouth Satellite', '18.0', '8', '318.0', '150.0', '3436.', '11.0', '70', 'US']
# ['AMC Rebel SST', '16.0', '8', '304.0', '150.0', '3433.', '12.0', '70', 'US']

parser.close()

# got a close...
# cleaning up...

# generator state

from inspect import getgeneratorstate


def gen(s):
    for c in s:
        yield c


g = gen("abc")
print(getgeneratorstate(g))  # GEN_CREATED

print(next(g))  # a
print(getgeneratorstate(g))  # GEN_SUSPENDED

print(list(g))  # ['b', 'c']
print(getgeneratorstate(g))  # GEN_CLOSED


def gen(s):
    for c in s:
        print(getgeneratorstate(global_gen))  # GEN_RUNNING
        yield c


global_gen = gen("abc")

next(global_gen)

from inspect import getgeneratorstate


def echo():
    while True:
        received = yield
        print(f"You said: {received}")


e = echo()
print(getgeneratorstate(e))  # GEN_CREATED

# e.send("python")  # TypeError: can't send non-None value to a just-started generator

next(e)
print(getgeneratorstate(e))  # GEN_SUSPENDED
e.send("python")  # You said: python

import math
from inspect import getgeneratorstate


def coroutine(gen_fn):
    def inner(*args, **kwargs):
        gen = gen_fn(*args, **kwargs)
        next(gen)
        return gen

    return inner


@coroutine
def power_up(p):
    result = None
    while True:
        received = yield result
        try:
            result = math.pow(received, p)
        except TypeError:
            result = None


squares = power_up(2)
cubes = power_up(3)
print(squares.send(2), cubes.send(2))  # 4.0 8.0
print(squares.send("a"))  # None


squares.close()
print(getgeneratorstate(squares))  # GEN_CLOSED
squares.send(3)  # StopIteration

from inspect import getgeneratorlocals, getgeneratorstate


def subgen():
    yield 1
    yield 2


def delegator():
    a = 100
    s = subgen()
    yield from s
    yield "subgen closed"


d = delegator()
print(getgeneratorstate(d), getgeneratorlocals(d))  # GEN_CREATED {}


next(d)
print(
    getgeneratorstate(d), getgeneratorlocals(d)
)  # GEN_SUSPENDED {'a': 100, 's': <generator object subgen at 0x000001E35BC5D2D0>}


s = getgeneratorlocals(d)["s"]
print(
    s, getgeneratorstate(s)
)  # <generator object subgen at 0x0000028337F8D2D0> GEN_SUSPENDED


print(
    next(d), getgeneratorstate(d), getgeneratorstate(s)
)  # 2 GEN_SUSPENDED GEN_SUSPENDED


print(
    next(d), getgeneratorstate(d), getgeneratorstate(s)
)  # subgen closed GEN_SUSPENDED GEN_CLOSED

try:
    next(d)
except StopIteration:
    pass

print(getgeneratorstate(d), getgeneratorstate(s))  # GEN_CLOSED GEN_CLOSED

# yield from

def is_iterable(item):
    try:
        iter(item)
    except Exception:
        return False
    else:
        return True


def flatten_gen(curr_item):
    if is_iterable(curr_item):
        for item in curr_item:
            yield from flatten_gen(item)
    else:
        yield curr_item


lst = [1, 2, [3, 4, [5, 6]], [7, [8, 9, 10]]]
print(list(flatten_gen(lst)))  # [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]

lst = ["abc", [1, 2, (3, 4)]]
# abc is is_iterable and a is is_iterable
print(list(flatten_gen(lst)))  # RecursionError: maximum recursion depth exceeded

def is_iterable(item, *, str_is_iterable=True):
    try:
        iter(item)
    except Exception:
        return False
    else:
        if isinstance(item, str):
            if str_is_iterable and len(item) > 1:
                return True
            else:
                return False
        else:
            return True


def flatten_gen(curr_item, *, str_is_iterable=True):
    if is_iterable(curr_item, str_is_iterable=str_is_iterable):
        for item in curr_item:
            yield from flatten_gen(item, str_is_iterable=str_is_iterable)
    else:
        yield curr_item


lst = [1, 2, [3, 4, [5, 6]], [7, [8, 9, 10]]]
print(list(flatten_gen(lst)))  # [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]

lst = ["abc", [1, 2, (3, 4)]]
print(list(flatten_gen(lst)))  # ['a', 'b', 'c', 1, 2, 3, 4]
print(list(flatten_gen(lst, str_is_iterable=False)))  # ['abc', 1, 2, 3, 4]

# defaultdict realize multidict

defaultdict 可以实现字典的值对应为列表或集合，列表会保持元素的插入顺序，集合会去除重复元素

from collections import defaultdict

d = defaultdict(list)
d['a'].append(1)
d['a'].append(3)
d['a'].append(2)
d['b'].append(2)
d['b'].append(2)
print(d)  # defaultdict(<class 'list'>, {'a': [1, 3, 2], 'b': [2, 2]})

d = defaultdict(set)
d['a'].add(1)
d['a'].add(3)
d['a'].add(2)
d['b'].add(2)
d['b'].add(2)
print(d)   # defaultdict(<class 'set'>, {'a': {1, 2, 3}, 'b': {2}})

# extend
d.setdefault('a', []).append(1)
d.setdefault('a', []).append(2)
d.setdefault('b', set()).add(2)
d.setdefault('b', set()).add(2)
print(d)   # {'a': [1, 2], 'b': {2}}

# zip and zip_longest

prices = {
    'ACME': 45.23,
    'AAPL': 612.78,
    'IBM': 205.55,
    'HPQ': 37.20,
    'FB': 10.75
}
print(sorted(zip(prices.values(), prices.keys())))

# zip() 函数创建的是只能返回一次的迭代器
prices_and_names = zip(prices.values(), prices.keys())
print(min(prices_and_names)) # OK (10.75, 'FB')
print(max(prices_and_names)) # ValueError: max() arg is an empty sequence


from itertools import zip_longest

l1 = ['a', 'b', 'c']
l2 = [10, 20, 30, 40]

print(list(zip_longest(l1, l2))) # [('a', 10), ('b', 20), ('c', 30), (None, 40)]
print(list(zip_longest(l1, l2, fillvalue="???"))) # [('a', 10), ('b', 20), ('c', 30), ('???', 40)]

l1 = [1, 2, 3, 4, 5, 6, 7, 8, 9]
l2 = ["a", "b", "c", "d"]

zip_result = [
    (item1, item2)
    for index1, item1 in enumerate(l1)
    for index2, item2 in enumerate(l2)
    if index1 == index2
]

print(zip_result) # [(1, 'a'), (2, 'b'), (3, 'c'), (4, 'd')]

# dict

persons = {
    "john": {"age": 20, "eye_color": "blue"},
    "jack": {"age": 25, "eye_color": "brown"},
    "jill": {"age": 22, "eye_color": "blue"},
    "eric": {"age": 35},
    "michael": {"age": 27},
}

eye_colors = {}
for person, details in persons.items():
    if "eye_color" in details:
        color = details["eye_color"]
    else:
        color = "unknown"

    if color in eye_colors:
        eye_colors[color].append(person)
    else:
        eye_colors[color] = [person]

print(eye_colors)  # {'blue': ['john', 'jill'], 'brown': ['jack'], 'unknown': ['eric', 'michael']}


eye_colors = {}
for person, details in persons.items():
    color = details.get("eye_color", "unknown")
    person_list = eye_colors.get(color, [])
    person_list.append(person)
    eye_colors[color] = person_list
print(eye_colors) # {'blue': ['john', 'jill'], 'brown': ['jack'], 'unknown': ['eric', 'michael']}


eye_colors = defaultdict(list)
for person, details in persons.items():
    color = details.get("eye_color", "unknown")
    eye_colors[color].append(person)

print(eye_colors)  # defaultdict(<class 'list'>, {'blue': ['john', 'jill'], 'brown': ['jack'], 'unknown': ['eric', 'michael']})

from collections import defaultdict
from functools import partial

persons = {
    "john": defaultdict(lambda: "unknown", age=20, eye_color="blue"),
    "jack": defaultdict(lambda: "unknown", age=20, eye_color="brown"),
    "jill": defaultdict(lambda: "unknown", age=22, eye_color="blue"),
    "eric": defaultdict(lambda: "unknown", age=35),
    "michael": defaultdict(lambda: "unknown", age=27),
}

eye_colors = defaultdict(list)
for person, details in persons.items():
    eye_colors[details["eye_color"]].append(person)

print(eye_colors) # defaultdict(<class 'list'>, {'blue': ['john', 'jill'], 'brown': ['jack'], 'unknown': ['eric', 'michael']})



# eyedict = lambda *args, **kwargs: defaultdict(lambda: "unknown", *args, **kwargs)
eyedict = partial(defaultdict, lambda: "unknown")

persons = {
    "john": eyedict(age=20, eye_color="blue"),
    "jack": eyedict(age=20, eye_color="brown"),
    "jill": eyedict(age=22, eye_color="blue"),
    "eric": eyedict(age=35),
    "michael": eyedict(age=27),
}


eye_colors = defaultdict(list)
for person, details in persons.items():
    eye_colors[details["eye_color"]].append(person)

print(eye_colors)  # defaultdict(<class 'list'>, {'blue': ['john', 'jill'], 'brown': ['jack'], 'unknown': ['eric', 'michael']})

# dict_keys and dict_items

字典的 keys () 和 items () 方法分别返回一个键视图对象和一个包含 (键，值) 对的元素视图对象，支持集合并、交、差运算，values () 也是类似，但是不支持集合运算，因为值视图不能保证元素互不相同。

a = {
    'x' : 1,
    'y' : 2,
    'z' : 3
}

b = {
    'w' : 10,
    'x' : 11,
    'y' : 2
}
print(a.keys(), a.items())      # dict_keys(['x', 'y', 'z']) dict_items([('x', 1), ('y', 2), ('z', 3)])
print(a.keys() & b.keys())      # {'x', 'y'}
print(a.keys() - b.keys())      # {'z'}
print(a.items() - b.items())    # {('x', 1), ('z', 3)}
c = {key: a[key] for key in a.keys() - {'z'}}
print(c)                        # {'x': 1, 'y': 2}

# dict & set comprehension

my_dict = {"Babb": 18, "Julian": 28}

reversed_dict = {value: key for key, value in my_dict.items()}
print(reversed_dict) # {18: 'Babb', 28: 'Julian'}

my_set = {key for key, value in my_dict.items()}
print(my_set) # {'Julian', 'Babb'}

my_set = set(my_dict.keys())
print(my_set) # {'Julian', 'Babb'}

# dict update

a: dict[str, int] = {"a": 1, "b": 2}
b: dict[str, int] = {"a": 2, "c": 3}

print(a | b) # {'a': 2, 'b': 2, 'c': 3}
a |= b
print(a) # {'a': 2, 'b': 2, 'c': 3}

# dict sort by values

Order is guaranteed since Python 3.7 in dictionaries.

1
2
3

scores: dict[str, int] = {"Babb": 101, "Tom": 42, "Alice": 34, "Bob": 68}
print(dict(sorted(scores.items(), key=lambda x: x[1]))) # {'Alice': 34, 'Tom': 42, 'Bob': 68, 'Babb': 101}
print(dict(sorted(scores.items(), key=lambda x: x[1], reverse=True))) # {'Babb': 101, 'Bob': 68, 'Tom': 42, 'Alice': 34}

# UserDict

from numbers import Real


class IntDict(dict):
    def __setitem__(self, key, value):
        if not isinstance(value, Real):
            raise ValueError("Value must be a real number.")
        super().__setitem__(key, value)

    def __getitem__(self, key):
        return int(super().__getitem__(key))


d = IntDict()

d["a"] = 10.5
print(d["a"])  # 10
print(d.get("x", "N/A"))  # N/A

# d["b"] = "python"  # ValueError: Value must be a real number.

d["b"] = 100.5
print(d.keys())  # dict_keys(['a', 'b'])
print(d.items())  # dict_items([('a', 10.5), ('b', 100.5)])


d.update({"c": 100.5})
print(d)  # {'a': 10.5, 'b': 100.5, 'c': 100.5}

d.update({"d": "python"})  # No error at here
print(d)  # {'a': 10.5, 'b': 100.5, 'c': 100.5, 'd': 'python'}

# print(d["d"])  # ValueError: invalid literal for int() with base 10: 'python'

from collections import UserDict
from numbers import Real


class IntDict(UserDict):
    def __setitem__(self, key, value):
        if not isinstance(value, Real):
            raise ValueError("Value must be a real number.")
        super().__setitem__(key, value)

    def __getitem__(self, key):
        return int(super().__getitem__(key))


d = IntDict()

d["a"] = 10.5
print(d["a"])  # 10
print(d.get("x", "N/A"))  # N/A

# d["b"] = "python"  # ValueError: Value must be a real number.

d["b"] = 100.5
print(d.keys())  # KeysView({'a': 10.5, 'b': 100.5})
print(d.items())  # ItemsView({'a': 10.5, 'b': 100.5})


d.update({"c": 100.5})
print(d)  # {'a': 10.5, 'b': 100.5, 'c': 100.5}

# d.update({"d": "python"})  # ValueError: Value must be a real number.

from collections import UserDict


class LimitedDict(UserDict):  # noqa: F821
    def __init__(self, keyset, min_value, max_value, *args, **kwargs):
        self._keyset = keyset
        self._min_value = min_value
        self._max_value = max_value
        super().__init__(*args, **kwargs)  # call overrided __setitem__

    def __setitem__(self, key, value):
        if key not in self._keyset:
            raise KeyError("Invalid key name")

        if not isinstance(value, int):
            raise ValueError("Value must be an integer type")

        if value < self._min_value or value > self._max_value:
            raise ValueError(
                f"Values must be between {self._min_value} and {self._max_value}"
            )

        super().__setitem__(key, value)


d = LimitedDict({"red", "green", "blue"}, 0, 255, red=10, green=10, blue=10)
print(d)  # {'red': 10, 'green': 10, 'blue': 10}


# d["purple"] = 10 # KeyError: 'Invalid key name'

# d["red"] = -10  # ValueError: Values must be between 0 and 255

# d["red"] = "red"  # ValueError: Value must be an integer type

# ChainMap

from collections import ChainMap


d1 = {"a": 1, "b": 2}
d2 = {"b": 20, "c": 3}
d3 = {"c": 30, "d": 4}

d = ChainMap(d1, d2, d3)
print(d)  # ChainMap({'a': 1, 'b': 2}, {'b': 20, 'c': 3}, {'c': 30, 'd': 4})
print(d["b"], d["c"])  # 2 3

for k, v in d.items():
    print(k, v)
# c 3
# d 4
# b 2
# a 1


# del d["c"] # KeyError: "Key not found in the first mapping: 'c'"


d1 = {"a": 1, "b": 2}
d2 = {"c": 3, "d": 4}
d = ChainMap(d1, d2)

d3 = {"d": 400, "e": 5}
d = d.new_child(d3)

print(d)  # ChainMap({'d': 400, 'e': 5}, {'a': 1, 'b': 2}, {'c': 3, 'd': 4})
print(d.parents)  # ChainMap({'a': 1, 'b': 2}, {'c': 3, 'd': 4})


d1 = {"a": 1, "b": 2}
d2 = {"c": 3, "d": 4}
d = ChainMap(d1, d2)
print(d.maps, type(d.maps))  # [{'a': 1, 'b': 2}, {'c': 3, 'd': 4}] <class 'list'>


d3 = {"e": 5, "f": 6}
d.maps.append(d3)
print(d)  # ChainMap({'a': 1, 'b': 2}, {'c': 3, 'd': 4}, {'e': 5, 'f': 6})

d.maps.insert(0, {"a": 100})
print(d)  # ChainMap({'a': 100}, {'a': 1, 'b': 2}, {'c': 3, 'd': 4}, {'e': 5, 'f': 6})


del d.maps[1]
print(d)  # ChainMap({'a': 100}, {'c': 3, 'd': 4}, {'e': 5, 'f': 6})

# MappingProxyType

from types import MappingProxyType


class Test:
    a = 100


print(repr(Test.__dict__))
# mappingproxy({'__module__': '__main__', '__firstlineno__': 1, 'a': 100, '__static_attributes__': (), '__dict__': <attribute '__dict__' of 'Test' objects>, '__weakref__': <attribute '__weakref__' of 'Test' objects>, '__doc__': None})

print(type(Test.__dict__), type(Test().__dict__))
# <class 'mappingproxy'> <class 'dict'>


d = {"a": 1, "b": 2}
mp = MappingProxyType(d)
print(
    mp.keys(),
    mp.values(),
    mp.items(),
    mp.get("a", "not found"),
    mp.get("A", "not found"),
)
# dict_keys(['a', 'b']) dict_values([1, 2]) dict_items([('a', 1), ('b', 2)]) 1 not found


# mp["c"] = 3  # TypeError: 'mappingproxy' object does not support item assignment


d["a"] = 100
d["c"] = "new item"
del d["b"]
print(d)  # {'a': 100, 'c': 'new item'}


print(mp, repr(mp))  # {'a': 100, 'c': 'new item'} mappingproxy({'a': 100, 'c': 'new item'})

# deque

from timeit import timeit
from collections import OrderedDict, deque


def create_ordereddict(n=100):
    d = OrderedDict()
    for i in range(n):
        d[str(i)] = i
    return d


def create_deque(n=100):
    return deque(range(n))


def pop_all_ordered_dict(n=1000, last=True):
    d = create_ordereddict(n)
    while True:
        try:
            d.popitem(last=last)
        except KeyError:
            # done popping
            break


def pop_all_deque(n=1000, last=True):
    dq = create_deque(n)
    if last:
        pop = dq.pop
    else:
        pop = dq.popleft

    while True:
        try:
            pop()
        except IndexError:
            break


n = 10_000
number = 1_000

results = dict()

results["dict_create"] = timeit(
    "create_ordereddict(n)", globals=globals(), number=number
)

results["deque_create"] = timeit("create_deque(n)", globals=globals(), number=number)

results["dict_create_pop_last"] = timeit(
    "pop_all_ordered_dict(n, last=True)", globals=globals(), number=number
)

results["dict_create_pop_first"] = timeit(
    "pop_all_ordered_dict(n, last=False)", globals=globals(), number=number
)

results["deque_create_pop_last"] = timeit(
    "pop_all_deque(n, last=True)", globals=globals(), number=number
)

results["deque_create_pop_first"] = timeit(
    "pop_all_deque(n, last=False)", globals=globals(), number=number
)

results["dict_pop_last"] = results["dict_create_pop_last"] - results["dict_create"]

results["dict_pop_first"] = results["dict_create_pop_first"] - results["dict_create"]

results["deque_pop_last"] = results["deque_create_pop_last"] - results["deque_create"]

results["deque_pop_first"] = results["deque_create_pop_first"] - results["deque_create"]

for key, result in results.items():
    print(f"{key}: {result}")


# dict_create: 1.252320499974303
# deque_create: 0.09276770002907142
# dict_create_pop_last: 1.8576560000074096
# dict_create_pop_first: 2.8178255000384524
# deque_create_pop_last: 0.37899299996206537
# deque_create_pop_first: 0.2397357000154443
# dict_pop_last: 0.6053355000331067
# dict_pop_first: 1.5655050000641495
# deque_pop_last: 0.28622529993299395
# deque_pop_first: 0.1469679999863729

# tuple

a = 1,
b = ()
c = (1,)
d = (1) # is like 2 * (1 + 1)
e = (1+1)
print(f"{type(a)=}")
print(f"{type(b)=}")
print(f"{type(c)=}")
print(f"{type(d)=}")
print(f"{type(e)=}")

# type(a)=<class 'tuple'>
# type(b)=<class 'tuple'>
# type(c)=<class 'tuple'>
# type(d)=<class 'int'>
# type(e)=<class 'int'>

# tuple compare

class A:
    def __init__(self, name):
        self.name = name

print((1, A('babb')) < (2, A('owen'))) # True
print((1, A('babb')) < (1, A('owen'))) # '<' not supported between instances of 'A' and 'A'
print((1, 1, A('babb')) < (1, 2, A('owen'))) # True
print((1, ['1', '2']) < (2, ['1', '3'])) # True

# typing namedtuple

from collections import namedtuple
from typing import NamedTuple

Point = namedtuple("Point", ["x", "y"])
p = Point(11, y=22)
print(p.x, p.y)
print(p[0], p[1])


class Color(NamedTuple):
    red: int
    green: int
    blue: int


c = Color(red=55, green=66, blue=77)
print(c.red, c.green, c.blue)
print(c[0], c[1], c[2])

# itertools

# itertools count

import itertools

counter = itertools.count(0, 2)

for i in counter:
    print(i)

    if i == 6:
        break
        
# 0
# 2
# 4
# 6

# itertools cycle

import itertools

letters = ["A", "B", "C"]
cycler = itertools.cycle(letters)

for i, letter in enumerate(cycler, start=1):
    print(f"{i}: {letter}")

    if i == 8:
        break

# 1: A
# 2: B
# 3: C
# 4: A
# 5: B
# 6: C
# 7: A
# 8: B

# itertools repeat

import itertools

string = "Hello"

repeater = itertools.repeat(string, 5)

for s in repeater:
    print(s)

# Hello
# Hello
# Hello
# Hello
# Hello

# itertools tee

import itertools

lst = [1, 2, 3]

tee = itertools.tee(lst, 3)
for it in tee:
    print(it, list(it))
    
# <itertools._tee object at 0x00000161C34E6C40> [1, 2, 3]
# <itertools._tee object at 0x00000161C34E6CC0> [1, 2, 3]
# <itertools._tee object at 0x00000161C34E6C80> [1, 2, 3]

# itertools accumulate

import itertools
import operator

numbers = [1, 2, 3, 4, 5]

accumulation = itertools.accumulate(numbers)
print(list(accumulation)) # [1, 3, 6, 10, 15]

accumulation = itertools.accumulate(numbers, operator.mul)
print(list(accumulation)) # [1, 2, 6, 24, 120]

# itertools chain

import itertools

a = [1, 2, 3]
b = ["a", "b", "c"]

combined = itertools.chain(a, b, a, b)
print(list(combined)) # [1, 2, 3, 'a', 'b', 'c', 1, 2, 3, 'a', 'b', 'c']

import itertools


def squares():
    print("yielding 1st item")
    yield (i**2 for i in range(2))
    print("yielding 2nd item")
    yield (i**2 for i in range(2, 4))
    print("yielding 3rd item")
    yield (i**2 for i in range(4, 6))


chained = itertools.chain(*squares())
for item in chained:
    print(item)
# yielding 1st item
# yielding 2nd item
# yielding 3rd item
# 0
# 1
# 4
# 9
# 16
# 25


chained = itertools.chain.from_iterable(squares())
for item in chained:
    print(item)
# yielding 1st item
# 0
# 1
# yielding 2nd item
# 4
# 9
# yielding 3rd item
# 16
# 25

def squares():
    print("yielding 1st item")
    yield (i**2 for i in range(2))
    print("yielding 2nd item")
    yield (i**2 for i in range(2, 4))
    print("yielding 3rd item")
    yield (i**2 for i in range(4, 6))


def chain_iterables(*iterables):
    for iterable in iterables:
        yield from iterable


def chain_from_iterable(iterable):
    for iterable in iterable:
        yield from iterable


for item in chain_iterables(*squares()):
    print(item)
# yielding 1st item
# yielding 2nd item
# yielding 3rd item
# 0
# 1
# 4
# 9
# 16
# 25

for item in chain_from_iterable(squares()):
    print(item)
# yielding 1st item
# 0
# 1
# yielding 2nd item
# 4
# 9
# yielding 3rd item
# 16
# 25

# itertools compress

import itertools

lst = ["a", "b", "c", "d"]
# selectors = [False, True, True]
selectors = [0, 1, 1]

compressed = itertools.compress(lst,selectors)
print(list(compressed)) # ['b', 'c']

# itertools dropwhile

import itertools

lst = [1, 2, 3, 4, 5, 1, 2]

remaining = itertools.dropwhile(lambda n: n < 3, lst)
print(list(remaining)) # [3, 4, 5, 1, 2]

# itertools takenwhile

import itertools

lst = [1, 2, 3, 4, 5, 1, 2]

taken = itertools.takewhile(lambda n: n < 3, lst)
print(list(taken)) # [1, 2]

# itertools filterfalse

import itertools
from typing import Any

lst = range(100)

filtered = itertools.filterfalse(lambda n: n % 10, lst)
print(list(filtered)) # [0, 10, 20, 30, 40, 50, 60, 70, 80, 90]


lst: list[Any] = [0, 1, False, True]

filtered = itertools.filterfalse(None, lst)
print(list(filtered)) # [0, False]

# itertools groupby

import itertools
from operator import itemgetter

lst = [1, 2, 2, 2, 2, 3, 3, 3, 1, 1]

grouped = [list(g) for _, g in itertools.groupby(lst)]
print(grouped) # [[1], [2, 2, 2, 2], [3, 3, 3], [1, 1]]

grouped = [list(g) for _, g in itertools.groupby(sorted(lst))]
print(grouped) # [[1, 1, 1], [2, 2, 2, 2], [3, 3, 3]]


employees = [
    [100, '犬夜叉', 2000],
    [200, '杀生丸', 3000],
    [100, '桔梗', 1000],
    [200, '玲', 800],
    [100, '冥加', 800],
    [300, '奈落', 4000]
]


employees = sorted(employees, key=itemgetter(0))
print(employees) # [[100, '犬夜叉', 2000], [100, '桔梗', 1000], [100, '冥加', 800], [200, '杀生丸', 3000], [200, '玲', 800], [300, '奈落', 4000]]

for no, item in itertools.groupby(employees, itemgetter(0)):
    print(list(item))

# 100 [[100, '犬夜叉', 2000], [100, '桔梗', 1000], [100, '冥加', 800]]
# 200 [[200, '杀生丸', 3000], [200, '玲', 800]]
# 300 [[300, '奈落', 4000]]

# itertools islice

import itertools

lst = ["a", "b", "c", "d", "e", "f"]

sliced = itertools.islice(lst, 2) # [0:2)
print(list(sliced)) # ['a', 'b']

sliced = itertools.islice(lst, 2, None) # [2:]
print(list(sliced)) # ['c', 'd', 'e', 'f']

import itertools


def colors():
    yield "red"
    yield "green"
    yield "blue"


color_generater = colors()
cycled = itertools.cycle(color_generater)
sliced = itertools.islice(cycled, 10)
print(list(sliced))
# ['red', 'green', 'blue', 'red', 'green', 'blue', 'red', 'green', 'blue', 'red']

1
2
3

with open("./test.csv") as f:
    for row in itertools.islice(f, 0, 20):
        print(row, end="")

# itertools starmap

import itertools
import operator

lst = [[1, 2], [10, 20], [100, 200]]
starmap_iterator = itertools.starmap(operator.mul, lst)
print(list(starmap_iterator))  # [2, 200, 20000]

lst = [[1, 2, 3], [10, 20, 30], [100, 200, 300]]
starmap_iterator = itertools.starmap(lambda x, y, z: x + y + z, lst)
print(list(starmap_iterator))  # [6, 60, 600]

import itertools
import operator


lst = [(2, 3), (2, 4), (2, 5)]
star_mapped = itertools.starmap(operator.pow, lst)
print(list(star_mapped)) # [8, 16, 32]


lst = [(1, 2, 3), (2, 4), (2, 5)]

def example(*args):
    temp = []
    for arg in args:
        temp.append(f"{arg}X")
    return temp

star_mapped = itertools.starmap(example, lst)
print(list(star_mapped)) # [['1X', '2X', '3X'], ['2X', '4X'], ['2X', '5X']]

# itertools zip_longest

import itertools

a = [1, 2, 3, 4, 5]
b = ["a", "b", "c"]
c = [True, False]

zipped = itertools.zip_longest(a, b, c)
print(list(zipped)) # [(1, 'a', True), (2, 'b', False), (3, 'c', None), (4, None, None), (5, None, None)]

zipped = itertools.zip_longest(a, b, c, fillvalue="Unknown")
print(list(zipped)) # [(1, 'a', True), (2, 'b', False), (3, 'c', 'Unknown'), (4, 'Unknown', 'Unknown'), (5, 'Unknown', 'Unknown')]

# itertools batched

from itertools import batched

data: list[int] = [1, 2, 3, 4, 5, 6, 7, 8, 9]
chunks: batched = batched(data, n=2)
print(chunks) # <itertools.batched object at 0x0000021A15886C40>
print(list(chunks)) # [(1, 2), (3, 4), (5, 6), (7, 8), (9,)]

chunks: batched = batched(data, n=3)
print(next(chunks)) # (1, 2, 3)
print(next(chunks)) # (4, 5, 6)
print(next(chunks)) # (7, 8, 9)

# itertools pairwise

import itertools

s = "abcde"
paired = itertools.pairwise(s)
print(list(paired)) # [('a', 'b'), ('b', 'c'), ('c', 'd'), ('d', 'e')]

l = range(1, 6)
paired = itertools.pairwise(l)
print(list(paired)) # [(1, 2), (2, 3), (3, 4), (4, 5)]

# itertools product

product 计算笛卡尔积。

100 以内所有勾股数，(3, 4, 5) 和 (4, 3, 5) 算不同的勾股数。

from itertools import product

for i in range(1, 100):
  for j in range(1, 100):
    for k in range(1, 100):
      if i * i + j * j == k  * k:
        print(i, j, k)

# product 计算笛卡尔积
items = product('ABCD', 'xy')
print(items)
for item in items:
  print(item)
# <itertools.product object at 0x000001BC20F1E600>
# ('A', 'x')
# ('A', 'y')
# ('B', 'x')
# ('B', 'y')
# ('C', 'x')
# ('C', 'y')
# ('D', 'x')
# ('D', 'y')


items = product(range(2), repeat=3)
print(items)
for item in items:
  print(item)
# <itertools.product object at 0x000001AAF4ECE6C0>
# (0, 0, 0)
# (0, 0, 1)
# (0, 1, 0)
# (0, 1, 1)
# (1, 0, 0)
# (1, 0, 1)
# (1, 1, 0)
# (1, 1, 1)

for i, j, k in product(range(1, 100), repeat=3):
  if i * i + j * j == k * k:
    print(i, j, k)

# itertools combinations_with_replacement

combinations_with_replacement 容器取 n 个元素，元素可以重复的组合。

100 以内所有勾股数，(3, 4, 5) 和 (4, 3, 5) 算相同的勾股数。

from itertools import combinations_with_replacement

for i in range(1, 100):
  for j in range(i, 100):
    for k in range(j, 100):
      if i * i + j * j == k  * k:
        print(i, j, k)

        
# combinations_with_replacement 容器取 n 个元素，元素可以重复的组合。
items = combinations_with_replacement('ABC', 2)
print(items)
for item in items:
  print(item)

# <itertools.combinations_with_replacement object at 0x000002845DA23EA0>
# ('A', 'A')
# ('A', 'B')
# ('A', 'C')
# ('B', 'B')
# ('B', 'C')
# ('C', 'C')

for i, j, k in combinations_with_replacement(range(1, 100), 3):
  if i * i + j * j == k * k:
    print(i, j, k)

# itertools combinations

combinations 容器取 n 个元素，元素不重复的组合。

有 10 个不同重量的砝码，任意取三个组合，总重量不在指定 [a, b] 重量区间的。

from itertools import combinations


weight_list = [5, 2, 3, 4, 5, 5, 6, 9, 12, 10]
n = len(weight_list)
a, b = 10, 30
weight_range = set(range(a, b + 1))

for i in range(n):
    for j in range(i + 1, n):
        for k in range(j + 1, n):
            total = weight_list[i] + weight_list[j] + weight_list[k]
            if total in weight_range:
                weight_range.remove(total)

print(weight_range)
# {29, 30}


# combinations 容器取 n 个元素，元素不重复的组合。
items = combinations("ABC", 2)
print(items)
for item in items:
    print(item)

# <itertools.combinations object at 0x000002150036EAE0>
# ('A', 'B')
# ('A', 'C')
# ('B', 'C')

for i, j, k in combinations(weight_list, 3):
    total = i + j + k
    if total in weight_range:
        weight_range.remove(total)

print(weight_range)
# {29, 30}

# itertools permutations

permutations 排列，每个顺序都称作一个排列。

SEND + MORE = MONEY, 每个字母代表不同的数字，S = ?, D = ? …

from itertools import permutations

# permutations 排列
items = permutations('ABC', 2)
print(items)
for item in items:
  print(item)
# <itertools.permutations object at 0x000002D06EB67E00>
# ('A', 'B')
# ('A', 'C')
# ('B', 'A')
# ('B', 'C')
# ('C', 'A')
# ('C', 'B')

items = permutations(range(3))
print(items)
for item in items:
  print(item)
# <itertools.permutations object at 0x000002D06E594E00>
# (0, 1, 2)
# (0, 2, 1)
# (1, 0, 2)
# (1, 2, 0)
# (2, 0, 1)
# (2, 1, 0)


for s, e, n, d, m, o, r, y in permutations(range(10), 8):
  if (s*1000+e*100+n*10+d) + (m*1000+o*100+r*10+e) == (m*10000+o*1000+n*100+e*10+y):
    if m != 0 and s != 0:
      print(f"{s=},{e=}, {n=}, {d=}, {m=}, {o=}, {r=}, {y=}")
# s=9,e=5, n=6, d=7, m=1, o=0, r=8, y=2
# 9567 + 1085 = 10652

# itertools accumulate

from functools import reduce
from itertools import accumulate
import operator


lst = [1, 2, 3, 4]

result = reduce(operator.mul, lst)
print(result)  # 24

result = accumulate(lst, operator.mul)
print(result)  # <itertools.accumulate object at 0x00000218E28AB100>
print(list(result))  # [1, 2, 6, 24]

result = accumulate(lst)
print(list(result))  # [1, 3, 6, 10]

import operator


def running_reduce(fn, iterable, start=None):
    it = iter(iterable)
    if start is None:
        acc = next(it)
    else:
        acc = start

    yield acc

    for item in it:
        acc = fn(acc, item)
        yield acc


lst = [1, 2, 3, 4]

result = running_reduce(operator.mul, lst)
print(list(result))  # [1, 2, 6, 24]

# exmaple

import itertools


def grid(min_val, max_val, step, *, num_dimensions=2):
    axis = itertools.takewhile(lambda x: x <= max_val, itertools.count(min_val, step))
    axes = itertools.tee(axis, num_dimensions)
    return itertools.product(*axes)


print(list(grid(-1, 1, 0.5)))
[
#     (-1, -1),
#     (-1, -0.5),
#     (-1, 0.0),
#     (-1, 0.5),
#     (-1, 1.0),
#     (-0.5, -1),
#     (-0.5, -0.5),
#     (-0.5, 0.0),
#     (-0.5, 0.5),
#     (-0.5, 1.0),
#     (0.0, -1),
#     (0.0, -0.5),
#     (0.0, 0.0),
#     (0.0, 0.5),
#     (0.0, 1.0),
#     (0.5, -1),
#     (0.5, -0.5),
#     (0.5, 0.0),
#     (0.5, 0.5),
#     (0.5, 1.0),
#     (1.0, -1),
#     (1.0, -0.5),
#     (1.0, 0.0),
#     (1.0, 0.5),
#     (1.0, 1.0),
# ]

# slice

iso_date_time = "2025-02-15T12:30:00.000Z"

date_slice = slice(0, 10)
time_slice = slice(11, 19)

print(iso_date_time[date_slice], iso_date_time[time_slice])

print(iso_date_time[slice(10)], iso_date_time[:10])
print(iso_date_time[slice(11, None)], iso_date_time[11:])

# hashable

k = [1, 2]
# my_dict = {k: "value"}  # This will raise a TypeError because lists are not hashable


class MyList(list):
    def __hash__(self):
        return id(self)

    def __eq__(self, other):
        return self is other


k = MyList([1, 2])
my_dict = {k: "value"}  # This will work because MyList is now hashable
k.append(3)
print(my_dict[k])


class MyClass:
    pass


k = MyClass()
my_dict = {
    k: "value"
}  # This will work because MyClass instances are hashable by default
print(my_dict[k])

# set membership is much faster than list or tuple membership

import string
import time

print(string.ascii_letters)

char_list = list(string.ascii_letters)
char_tuple = tuple(string.ascii_letters)
char_set = set(string.ascii_letters)


def membership_test(n, container):
    for _ in range(n):
        if "z" in container:
            pass


start = time.perf_counter()
membership_test(10000000, char_list)
end = time.perf_counter()
print("list", end - start)

start = time.perf_counter()
membership_test(10000000, char_tuple)
end = time.perf_counter()
print("tuple", end - start)

start = time.perf_counter()
membership_test(10000000, char_set)
end = time.perf_counter()
print("set", end - start)

# list 2.0961915999941994
# tuple 2.1027070999844
# set 0.18551109998952597

# enumerate start

letters: str = "ABCD"
    
for i, letter in enumerate(letters):
    print(f"{i}: {letter}")
    
# 0: A
# 1: B
# 2: C
# 3: D

for i, letter in enumerate(letters, start=1):
    print(f"{i}: {letter}")
    
# 1: A
# 2: B
# 3: C
# 4: D

# Counter

from collections import Counter


letters: list[str] = ["A", "A", "A", "B", "C", "C", "D"]
counter: Counter = Counter(letters)

print(counter) # Counter({'A': 3, 'C': 2, 'B': 1, 'D': 1})
print(counter.total()) # 7
print(counter.most_common()) # [('A', 3), ('C', 2), ('B', 1), ('D', 1)]
print(counter.most_common(n=2)) # [('A', 3), ('C', 2)]


c1 = Counter(a=1, b=2, c=3)
c2 = Counter(b=1, c=2, d=3)
c1.update(c2)
print(c1) # Counter({'c': 5, 'b': 3, 'd': 3, 'a': 1})

c1 = Counter(a=1, b=2, c=3)
c2 = Counter(b=1, c=2, d=3)
c1.subtract(c2)
print(c1) # Counter({'a': 1, 'b': 1, 'c': 1, 'd': -3})

c1 = Counter('aabbccddee')
print(c1) # Counter({'a': 2, 'b': 2, 'c': 2, 'd': 2, 'e': 2})
c1.update('abcdef')
print(c1) # Counter({'a': 3, 'b': 3, 'c': 3, 'd': 3, 'e': 3, 'f': 1})


c1 = Counter('aabbcc')
c2 = Counter('abc')
print(c1, c2)  # Counter({'a': 2, 'b': 2, 'c': 2}) Counter({'a': 1, 'b': 1, 'c': 1})
print(c1 + c2) # Counter({'a': 3, 'b': 3, 'c': 3})
print(c1 - c2) # Counter({'a': 1, 'b': 1, 'c': 1})
# &: keeps the minimum of the key values
print(c1 & c2) # Counter({'a': 1, 'b': 1, 'c': 1})
# |: keeps the maximum of the key values
print(c1 | c2) # Counter({'a': 2, 'b': 2, 'c': 2})

c1 = Counter(a=10, b=-10)
# The unary + can also be used to remove any non-positive count from the Counter
print(+c1) # Counter({'a': 10})
# The unary - changes the sign of each counter, and removes any non-positive result
print(-c1) # Counter({'b': 10})

# TypedDict

from typing import Required, TypedDict, NotRequired

class Coordinate(TypedDict):
    x: float
    y: float
    label: str
    category: NotRequired[str]


coordinate: Coordinate = {"x": 10, "y": 10, "label": "Profit", "category": "Finance"}
coordinate2: Coordinate =  {"x": 15, "y": 25, "label": "Expenses"}


Vote = TypedDict("Vote", {"for": int, "against": int}, total=True)
vote: Vote = {"for": 100, "against": 200}
# vote: Vote = {"for": 100}  # "against" is required in "Vote"

Vote = TypedDict("Vote", {"for": int, "against": int}, total=False)
vote2: Vote = {"for": 100}

Vote = TypedDict("Vote", {"for": int, "against": Required[int]}, total=False)
vote3: Vote = {"for": 100, "against": 200}
# vote3: Vote = {"for": 100} # "against" is required in "Vote"

# itemgetter

from operator import itemgetter

elements: list[int] = [1, 2, 3, 4, 5]
first_and_last: itemgetter = itemgetter(0, -1, 2)
print(first_and_last(elements)) # (1, 5, 3)

items: dict[str, int] = {"a": 1, "b": 2, "c": 3, "d": 4}
a_and_c: itemgetter = itemgetter("a", "c")
print(a_and_c(items)) # (1, 3)

# remove list duplicates and keep order

numbers: list[int] = [1, 1, 3, 3, 3, 2,
                      2, 1, 2, 3, 4, 4]

print(list(set(numbers))) # [1, 2, 3, 4] the order not keep

print(dict.fromkeys(numbers)) # {1: None, 3: None, 2: None, 4: None}
# Order is guaranteed since Python 3.7 in dictionaries.
print(list(dict.fromkeys(numbers))) # [1, 3, 2, 4]

# reversed

reversed return an iterator, if the given sequence is too big, can consider using reversed.

name: str = "Babb"

print(name[::-1], "".join(reversed(name))) # bbaB bbaB

print(reversed(name)) # <reversed object at 0x000002B4CAA4F580>
reverse_iterator = reversed(name)
print(next(reverse_iterator)) # b
for item in reverse_iterator:
    print(item)
# b
# a
# B

# flatten list

from typing import Any, Callable

type Flatten = Callable[[list], list]

flatten: Flatten = lambda target: sum((flatten(sub) if isinstance(sub, list) else [sub] for sub in target), start=[])
nested_list: list[Any] = [1, [2, [3, 4]], ["a", ["b", ["c", "d"]]]]
print(flatten(nested_list)) # [1, 2, 3, 4, 'a', 'b', 'c', 'd']

print([1] + [2] + []) # [1, 2]
print(sum([[1], [2]], start=[])) # [1, 2]

# secrets password generator

import secrets
import string
from typing import Callable

password_generator: Callable[[int], str] = lambda n: "".join(secrets.choice(string.ascii_letters + string.digits + string.punctuation) for _ in range(n))
print(password_generator(5))
print(password_generator(10))
print(password_generator(20))
# cGWN3
# i?k6t5wYYT
# lyOg_u*w8ofT;vN$=6=*

# sorted

import random


lst = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
print(sorted(lst, key=lambda _: random.random())) # [5, 6, 3, 9, 10, 8, 4, 2, 7, 1]

# random

# random choices

from collections import namedtuple
import random


lst = list(range(1000))
choices_result = random.choices(lst, k=5)
print(choices_result)  # [984, 594, 941, 706, 542]


lst = ["a", "b", "c"]
choices_result = random.choices(lst, k=5, weights=[10, 1, 1])
print(choices_result)  # ['a', 'b', 'a', 'a', 'a']


Freq = namedtuple("Freq", "count freq")


def freq_counts(lst):
    total = len(lst)
    return {k: Freq(lst.count(k), 100 * lst.count(k) / total) for k in set(lst)}


lst = ["a", "b", "c"]
result = freq_counts(random.choices(lst, k=1000))
print(result)  # {'c': Freq(count=346, freq=34.6), 'a': Freq(count=338, freq=33.8), 'b': Freq(count=316, freq=31.6)}


random.seed(0)
result = freq_counts(random.choices(lst, k=1000, weights=(8, 1, 1)))
print(result)  # {'b': Freq(count=86, freq=8.6), 'a': Freq(count=810, freq=81.0), 'c': Freq(count=104, freq=10.4)}


random.seed(0)
result = freq_counts(random.choices(lst, k=1000, cum_weights=(8, 9, 10)))
print(result)  # {'a': Freq(count=810, freq=81.0), 'b': Freq(count=86, freq=8.6), 'c': Freq(count=104, freq=10.4)}

import random
from time import perf_counter

denominators = random.choices([0, 1], k=1_000_000)

start = perf_counter()
for denominator in denominators:
    if denominator == 0:
        continue
    else:
        10 / denominator
end = perf_counter()
print(
    f"Avg elapsed time: {(end - start) / len(denominators)}"
)  # Avg elapsed time: 4.421700001694262e-08


start = perf_counter()
for denominator in denominators:
    try:
        10 / denominator
    except ZeroDivisionError:
        continue

end = perf_counter()
print(
    f"Avg elapsed time: {(end - start) / len(denominators)}"
)  # Avg elapsed time: 1.2862039997708052e-07


denominators = random.choices([0, 1], k=1_000_000, weights=[1, 9])

start = perf_counter()
for denominator in denominators:
    if denominator == 0:
        continue
    else:
        10 / denominator
end = perf_counter()
print(
    f"Avg elapsed time: {(end - start) / len(denominators)}"
)  # Avg elapsed time: 5.262949998723343e-08


start = perf_counter()
for denominator in denominators:
    try:
        10 / denominator
    except ZeroDivisionError:
        continue

end = perf_counter()
print(
    f"Avg elapsed time: {(end - start) / len(denominators)}"
)  # Avg elapsed time: 5.35208000219427e-08

# random sample

import random
from collections import Counter

lst = range(20)

random.seed(0)
# Chooses k unique random elements from a population sequence.
result = random.sample(lst, k=10)
print(result)  # [12, 13, 1, 8, 15, 6, 19, 4, 7, 5]

random.seed(0)
# Return a k sized list of population elements chosen with replacement.
result = random.choices(lst, k=10)
print(result)  # [16, 15, 8, 5, 10, 8, 15, 6, 9, 11] two 15

import random

suits = "C", "D", "H", "A"
ranks = tuple(range(2, 11)) + tuple("JQKA")

deck = [str(rank) + suit for suit in suits for rank in ranks]
print(deck)  # ['2C', '3C', '4C', '5C', '6C', '7C', '8C', '9C', '10C', 'JC', 'QC', 'KC', 'AC', '2D', '3D', '4D', '5D', '6D', '7D', '8D', '9D', '10D', 'JD', 'QD', 'KD', 'AD', '2H', '3H', '4H', '5H', '6H', '7H', '8H', '9H', '10H', 'JH', 'QH', 'KH', 'AH', '2A', '3A', '4A', '5A', '6A', '7A', '8A', '9A', '10A', 'JA', 'QA', 'KA', 'AA']

result = Counter(random.sample(deck, k=20))
print(result)  # Counter({'5D': 1, '10C': 1, 'KC': 1, 'JD': 1, '10A': 1, '7A': 1, 'QA': 1, '6A': 1, '6C': 1, 'KA': 1, '4D': 1, '8H': 1, '7D': 1, 'JA': 1, 'JC': 1, '4A': 1, '4C': 1, '10H': 1, 'KD': 1, '3D': 1})

result = Counter(random.choices(deck, k=20))
print(result)  # Counter({'5H': 2, '3C': 2, '2D': 2, 'KH': 2, '10C': 1, '6A': 1, 'AH': 1, '7H': 1, 'AC': 1, '7A': 1, '2C': 1, '4A': 1, '9H': 1, 'QC': 1, '2A': 1, '5C': 1})

# Class

# super()

super 用来调用父类某个已经覆盖的方法，遵循菱形继承顺序，格式 super([type[, object-or-type]]) , 在 python3 中可以省略参数 super() , 在 python2 中使用 super(subtype, self) 。

mro() stands for Method Resolution Order. It returns a list of types the class is derived from, in the order they are searched for methods.

class A:
    def spam(self):
        print('A.spam')
        super().spam() # 会调用 B 的 spam 方法，尽管 A 没有直接继承 B

class B:
    def spam(self):
        print('B.spam')

class C(A, B):
    pass

c = C()
c.spam()

print(C.mro())
print(C.__mro__)
# (<class '__main__.C'>, <class '__main__.A'>, <class '__main__.B'>, <class 'object'>)

# class point to class

In [12]: class A:
    ...:     pass

In [13]: a = A()

In [14]: a.__class__
Out[14]: __main__.A

In [15]: b = a.__class__()

In [16]: b
Out[16]: <__main__.A at 0x4fcb5f8>

In [17]: b.__class__
Out[17]: __main__.A

# calendar

import calendar
# 打印日历
cal = calendar.month(2018, 9)
print(cal)
#    September 2018
# Mo Tu We Th Fr Sa Su
#                 1  2
#  3  4  5  6  7  8  9
# 10 11 12 13 14 15 16
# 17 18 19 20 21 22 23
# 24 25 26 27 28 29 30


# 判断是否为闰年
ret = calendar.isleap(2018)
print(ret)
# False

# type() and isinstance

class A:
    pass


class B(A):
    pass


print(isinstance(A(), A))  # True
print(type(A()) is A)  # True

print(isinstance(B(), A))  # True
print(type(B()) is A)  # False

print(isinstance(1, (float, int)))  # True, isinstance 第二个参数可以是一个元组

# typing udt(user-defined types)

from typing import List

Vector = List[float]
Matrix = List[Vector]

def addMatrix(a : Matrix, b : Matrix) -> Matrix:
    result = []

    for i, row in enumerate(a):
        result_row =[]

        for j, _ in enumerate(row):
            result_row += [a[i][j] + b[i][j]]

        result += [result_row]
    return result

x = [[1.0, 0.0], [0.0, 1.0]]
y = [[2.0, 1.0], [0.0, -2.0]]
z = addMatrix(x, y)

print(z)
# [[3.0, 1.0], [0.0, -1.0]]

# dynamic monkey patch

class Foo():
    def bar(self):
        print('Foo.bar')

def bar(self):
    print('Modified bar')

Foo().bar() # Foo.bar
Foo.bar = bar
Foo().bar() # Modified bar

# dynamic import

动态导入是指模块可以以字符串的形式导入

module_names = ['sys', 'os', 're']
modules = list(map(__import__, module_names))
sys = modules[0]
print(sys.version)


import importlib
module2 = importlib.import_module('sys')
print(module2.version)

# import order

import sys

# 查看模块搜索顺序
print(sys.path)

# 程序执行时添加新的模块路径
sys.path.append("/home/jbn/xxx")

# 可以确保先搜索这个路径
sys.path.insert(0, "/home/jbn/xxx")
print(sys.path)

# relative and absolute import

Absolute import is from the sys.path.

Relative import is from the __package__ , if one file as the start script, the __package__ is None.

# dataclass

from dataclasses import dataclass, field
from datetime import datetime
import random
from typing import ClassVar


@dataclass(order=True)
class Person:
    name: str
    age: int
    height: int = field(
        default_factory=lambda: random.randint(150, 200), repr=False
    )  # Default height in cm

    count: ClassVar[int] = 0  # Class variable to count instances

    def __post_init__(self):
        Person.count += 1


p1 = Person(name="Alice", age=30)
p2 = Person(name="Alice", age=30)

print(p1 == p2)  # True, because dataclass provides an automatic __eq__ method
print(p1)  # Person(name='Alice', age=30), automatic __repr__
print(p1.name)  # Alice

p3 = Person(name="Bob", age=25)
print(p1 < p3)  # True, because of the order=True parameter

print(p1.height)  # Random height between 150 and 200

print(Person.count)  # Number of Person instances created


@dataclass
class AppConfig:
    debug: bool = False
    database_url: str = "sqlite:///:memory:"
    max_connections: int = 10


@dataclass
class APIResponse:
    status_code: int
    data: dict
    headers: dict = field(default_factory=dict)
    success: bool = field(init=False)
    timestamp: datetime = field(default_factory=datetime.now)

    def __post_init__(self):
        self.success = 200 <= self.status_code < 300

frozen=True

from dataclasses import dataclass

@dataclass(frozen=True)
class Fruit:
    name: str
    calories: float = 10

banana = Fruit("Banana")
print(banana)

banana.calories = 30 # dataclasses.FrozenInstanceError: cannot assign to field 'calories'

slots=True

from dataclasses import dataclass

@dataclass(slots=True)
class Fruit:
    name: str
    calories: float


apple = Fruit("Apple", 20)
print(apple)
print(apple.__dict__) # AttributeError: 'Fruit' object has no attribute '__dict__'

kw_only=True

from dataclasses import dataclass

@dataclass(kw_only=True)
class Fruit:
    name: str
    calories: float


apple = Fruit(name="Apple", calories=20)
print(apple)
# apple = Fruit("Apple", 20) # TypeError: Fruit.__init__() takes 1 positional argument but 3 were given

order=True

from dataclasses import dataclass

@dataclass(order=True)
class Fruit:
    name: str
    calories: float


apple = Fruit("Apple", 20)
orange = Fruit("Orange", 20)
print(apple < orange) # True

__post_init__

from dataclasses import InitVar, dataclass, field

@dataclass
class Fruit:
    name: str
    grams: float
    cost_per_kg: float
    is_rare: InitVar[bool] = False

    # Caculate later
    total_value: float = field(init=False)

    def __post_init__(self, is_rare: bool) -> None:
        self.total_value = (self.grams / 1000) * self.cost_per_kg

        if is_rare:
            self.total_value *= 2
            self.cost_per_kg *= 2


apple: Fruit = Fruit("Apple", 2500, 3)
print(apple) # Fruit(name='Apple', grams=2500, cost_per_kg=3, total_value=7.5)

orange: Fruit = Fruit("Orange", 2000, 3, is_rare=True)
print(orange) # Fruit(name='Orange', grams=2000, cost_per_kg=6, total_value=12.0)

field default_factory

import random
import string
from dataclasses import dataclass, field

@dataclass(kw_only=True)
class Person:
    name: str
    address: str
    active: bool = True
    # email_addresses: list[str] = [] #ValueError: mutable default <class 'list'> for field email_addresses is not allowed: use default_factory
    email_addresses: list[str] = field(default_factory=list)
    id: str = field(init=False, default_factory=lambda: "".join(random.choices(string.ascii_uppercase, k=12)))
    search_string: str = field(init=False, repr=False)

    def __post_init__(self) -> None:
        self.search_string = f"{self.name} {self.address}"

person = Person(name="Babb", address="Newyork")
print(person) # Person(name='Babb', address='Newyork', active=True, email_addresses=[], id='MTHHVWTOYFOE')
person = Person(name="Babb", active=False, address="Newyork", email_addresses=["babb@abc.com", "babb@xyz.com"])
print(person) # Person(name='Babb', address='Newyork', active=False, email_addresses=['babb@abc.com', 'babb@xyz.com'], id='ZQTETEJQJAJO')

# property and descriptor

from math import pi


class Circle:
    def __init__(self, radius):
        self.radius = radius

    @property
    def radius(self):
        return self._radius

    @radius.setter
    def radius(self, value):
        if not (isinstance(value, (int, float)) and value > 0):
            raise ValueError("Radius is required to be a positive number!")
        self._radius = value

    @radius.deleter
    def radius(self):
        del self._radius

    @property
    def area(self):
        return pi * self.radius * self.radius


c = Circle(1)
print(c.area)
c.radius = 2
print(c.area)


# class Rectange:
#     def __init__(self, width, height):
#         self.width = width
#         self.height = height

#     @property
#     def width(self):
#         return self.width

#     @width.setter
#     def width(self, value):
#         if not (isinstance(value, (int, float)) and value > 0):
#             raise ValueError("Width must be positive.")
#         self._width = value

#     @property
#     def height(self):
#         return self.height

#     @height.setter
#     def height(self, value):
#         if not (isinstance(value, (int, float)) and value > 0):
#             raise ValueError("Height must be positive.")
#         self._height = value


# class PositiveNumber:
#     def __get__(self, instance, owner):
#         return 1

#     # def __set__(self, instance, value):
#     #     pass


# class Rectangle:
#     width = PositiveNumber()

#     def __init__(self, width, height):
#         self.width = width
#         self.height = height


# r = Rectangle(2, 2)
# print(r.width)


# class PositiveNumber:
#     def __get__(self, instance, owner):
#         return instance._width

#     def __set__(self, instance, value):
#         if not (isinstance(value, (int, float)) and value > 0):
#             raise ValueError("Width must be positive.")
#         instance._width = value


# class Rectangle:
#     width = PositiveNumber()

#     def __init__(self, width, height):
#         self.width = width
#         self.height = height


class PositiveNumber:
    def __set_name__(self, owner, name):
        self.private_name = f"_{name}"

    def __get__(self, instance, owner):
        return getattr(instance, self.private_name)

    def __set__(self, instance, value):
        if not (isinstance(value, (int, float)) and value > 0):
            raise ValueError(f"{self.private_name[1:].capitalize()} must be positive.")
        setattr(instance, self.private_name, value)


class Rectangle:
    width = PositiveNumber()
    height = PositiveNumber()

    def __init__(self, width, height):
        self.width = width
        self.height = height


r = Rectangle(2, 2)
print(r.width, r.height)
r.height = -1
r.width = -1

class ValidString:
    def __init__(self, min_length):
        self.min_length = min_length

    def __set_name__(self, owner_class, property_name):
        self.property_name = property_name

    def __set__(self, instance, value):
        if not isinstance(value, str):
            raise ValueError(f"{self.property_name} must be a string.")
        if len(value) < self.min_length:
            raise ValueError(
                f"{self.property_name} must be at least {self.min_length} characters"
            )
        instance.__dict__[self.property_name] = value

    def __get__(self, instance, owner_class):
        if instance is None:
            return self
        else:
            print(f"calling __get__ for {self.property_name}")
            return instance.__dict__.get(self.property_name, None)


class Person:
    first_name = ValidString(3)
    last_name = ValidString(3)


p = Person()
try:
    p.first_name = "Alex"
    p.last_name = "M"
except ValueError as ex:
    print(ex)
# last_name must be at least 3 characters

print(p.__dict__)  # {'first_name': 'Alex'}

import numbers


class ValidType:
    def __init__(self, type_):
        self._type = type_

    def __set_name__(self, owner_class, prop_name):
        self.prop_name = prop_name

    def __set__(self, instance, value):
        if not isinstance(value, self._type):
            raise ValueError(f"{self.prop_name} must be of type {self._type.__name__}")
        instance.__dict__[self.prop_name] = value

    def __get__(self, instance, owner_class):
        if instance is None:
            return self
        else:
            return instance.__dict__.get(self.prop_name, None)


class Person:
    age = ValidType(int)
    height = ValidType(float)
    tags = ValidType(list)
    favorite_foods = ValidType(tuple)
    name = ValidType(str)


p = Person()

try:
    p.age = 10.5
except ValueError as ex:
    print(ex)
# age must be of type int


try:
    p.height = 10
except ValueError as ex:
    print(ex)
# height must be of type float

print(isinstance(10.1, numbers.Real), isinstance(10, numbers.Real))  # True True

class Person:
    age = ValidType(int)
    height = ValidType(numbers.Real)
    tags = ValidType(list)
    favorite_foods = ValidType(tuple)
    name = ValidType(str)


p = Person()
p.height = 10
print(p.height)  # 10

property is descriptor

from numbers import Integral


class Person:
    def get_age(self):
        return getattr(self, "_age", None)

    def set_age(self, value):
        if not isinstance(value, Integral):
            raise ValueError("age: must be an integer.")
        if value < 0:
            raise ValueError("age: must be a non-negative integer.")
        self._age = value

    age = property(fget=get_age, fset=set_age)


p = Person()
try:
    p.age = -10
except ValueError as ex:
    print(ex)
# age: must be a non-negative integer.

prop = Person.age
print(hasattr(prop, "__set__"), hasattr(prop, "__get__"))  # True True

from numbers import Integral

class Person:
    @property
    def age(self):
        return getattr(self, '_age', None)
    
    @age.setter
    def age(self, value):
        if not isinstance(value, Integral):
            raise ValueError('age: must be an integer.')
        if value < 0:
            raise ValueError('age: must be a non-negative integer.')
        self._age = value
        
class Person:
    def get_age(self):
        return getattr(self, '_age', None)
    
    age = property(get_age)

    def set_age(self, value):
        if not isinstance(value, Integral):
            raise ValueError('age: must be an integer.')
        if value < 0:
            raise ValueError('age: must be a non-negative integer.')
        self._age = value

    age = age.setter(set_age)

function and method is descriptor

import sys


def add(a, b):
    return a + b


print(hasattr(add, "__get__"), hasattr(add, "__set__"))  # True False


owener = sys.modules["__main__"]
fn = add.__get__(None, owener)
print(fn is add)  # True


class Person:
    def __init__(self, name):
        self.name = name

    def say_hello(self):
        return f"{self.name} says hello"


person = Person("Babb")
bound_method = Person.say_hello.__get__(person, Person)
print(bound_method) # <bound method Person.say_hello of <__main__.Person object at 0x00000194F1BE7230>>
print(bound_method())  # Babb says hello

print(person.say_hello)  # <bound method Person.say_hello of <__main__.Person object at 0x0000018E37587230>>
print(Person.say_hello)  # <function Person.say_hello at 0x000001CFBE00D120>
print(person.say_hello.__func__ is Person.say_hello)  # True

import types


class MyFunc:
    def __init__(self, func):
        self._func = func

    def __get__(self, instance, owner):
        if instance is None:
            # called from class
            print("__get__ called from class")
            return self._func
        else:
            # called from instance
            print("__get__ called from an instance")
            return types.MethodType(self._func, instance)


def hello(self):
    print(f"{self.name} says hello!")


class Person:
    def __init__(self, name):
        self.name = name

    say_hello = MyFunc(hello)


print(Person.say_hello)
# __get__ called from class
# <function hello at 0x000001BE35063C40>

p = Person("Babb")
print(p.say_hello)
# __get__ called from class
# <function hello at 0x000001BE35063C40>


print(p.say_hello.__func__)
# __get__ called from an instance
# <function hello at 0x0000018D71383C40>

# enum

from enum import Enum, IntEnum, StrEnum, auto


class Color(Enum):
    RED = 1
    GREEN = 2
    BLUE = 3

    @classmethod
    def print_members(cls):
        for color in cls:
            print(f"{color.name} = {color.value}")


Color.print_members()
print(type(Color.RED))  # <enum 'Color'>
print(Color.RED.name)  # RED
print(Color.RED.value)  # 1
print(Color.RED == Color(1))  # True
print(Color.RED == 1)  # False


class IntColor(IntEnum):
    RED = 1
    GREEN = 2
    BLUE = 3

    @classmethod
    def print_members(cls):
        for color in cls:
            print(f"{color.name} = {color.value}")


IntColor.print_members()
print(IntColor.RED == 1)  # True


class StrColor(StrEnum):
    RED = "red"
    GREEN = "green"
    BLUE = "blue"

    @classmethod
    def print_members(cls):
        for color in cls:
            print(f"{color.name} = {color.value}")


StrColor.print_members()
print(StrColor.RED == "red")  # True


class AutoColor(StrEnum):
    RED = auto()
    GREEN = auto()
    BLUE = auto()

    @classmethod
    def print_members(cls):
        for color in cls:
            print(f"{color.name} = {color.value}")


AutoColor.print_members()


class HTTPStatus(IntEnum):
    OK = 200
    NOT_FOUND = 404
    INTERNAL_SERVER_ERROR = 500


def handle_http_status(status: HTTPStatus):
    if status == HTTPStatus.OK:
        return "Success"
    elif status == HTTPStatus.NOT_FOUND:
        return "Not Found"
    elif status == HTTPStatus.INTERNAL_SERVER_ERROR:
        return "Server Error"
    else:
        return "Unknown Status"


class Light:
    def __init__(self, color: Color):
        self.color = color


light = Light(Color.RED)
print(light.color)

enum aliases

from enum import Enum

class Color(Enum):
    red = 1
    crimson = 1
    carmine = 1
    blue = 2
    aquamarine = 2

print(list(Color)) # [<Color.red: 1>, <Color.blue: 2>]
print(Color.crimson, Color.carmine, Color.aquamarine) # Color.red Color.red Color.blue
print(Color.carmine is Color.red) # True
print(Color(1), Color["crimson"]) # Color.red Color.red
print(Color.crimson in Color, 1 in Color) # True True
print(Color.__members__) # {'red': <Color.red: 1>, 'crimson': <Color.red: 1>, 'carmine': <Color.red: 1>, 'blue': <Color.blue: 2>, 'aquamarine': <Color.blue: 2>}

# Ensuring No Aliases
import enum

# ValueError: duplicate values found in <enum 'Color'>: crimson -> red, carmine -> red, aquamarine -> blue
@enum.unique
class Color(Enum):
    red = 1
    crimson = 1
    carmine = 1
    blue = 2
    aquamarine = 2

Extending Custom Enumerations

We can also extend (subclass) our custom enumerations - but only under certain circumstances: as long as the enumeration we are extending does not define any members:

class Color(Enum):
    RED = 1
    GREEN = 2
    BLUE = 3
    
try:
    class ColorAlpha(Color):
        ALPHA = 4
except TypeError as ex:
    print(ex) # Cannot extend enumerations

But this would work:

class ColorBase(Enum):
    def hello(self):
        return f'{str(self)} says hello!'
    
class Color(ColorBase):
    RED = 'red'
    GREEN = 'green'
    BLUE = 'blue'
    
print(Color.RED.hello()) # Color.RED says hello!

Custom OrderedEnum

from enum import Enum
from functools import total_ordering

@total_ordering
class OrderedEnum(Enum):
    """Creates an ordering based on the member values. 
    So member values have to support rich comparisons.
    """
    
    def __lt__(self, other):
        if isinstance(other, OrderedEnum):
            return self.value < other.value
        return NotImplemented

class Number(OrderedEnum):
    ONE = 1
    TWO = 2
    THREE = 3
    
class Dimension(OrderedEnum):
    D1 = 1,
    D2 = 1, 1
    D3 = 1, 1, 1

print(Number.ONE < Number.THREE, Number.ONE >= Number.ONE) # True True
print(Dimension.D1 < Dimension.D3, Dimension.D1 >= Dimension.D2) # True False

Custom TwoValueEnum

class TwoValueEnum(Enum):
    def __new__(cls, member_value, member_phrase):
        # create a new instance of cls
        member = object.__new__(cls)
        
        # set up instance attributes
        member._value_ = member_value  # Set the value property - Enum has a special class attribute we can use, called _value_.
        member.phrase = member_phrase
        return member
    
class AppStatus(TwoValueEnum):
    OK = (0, 'No Problem!')
    FAILED = (1, 'Crap!')

print(AppStatus.FAILED, AppStatus.FAILED.name, AppStatus.FAILED.value, AppStatus.FAILED.phrase) # AppStatus.FAILED FAILED 1 Crap!

# abstract class

from abc import ABC, abstractmethod


class Course(ABC):
    @abstractmethod
    def enroll(self, student):
        print(f"{student} enrolled in the Course.")


class MathCourse(Course):
    def enroll(self, student):
        print(f"{student} enrolled in the Math Course.")


class HistoryCourse(Course):
    def enroll(self, student):
        print(f"{student} enrolled in the History Course.")


courses = [MathCourse(), HistoryCourse()]
for course in courses:
    course.enroll("Alice")
    course.enroll("Bob")

# Alice enrolled in the Math Course.
# Bob enrolled in the Math Course.
# Alice enrolled in the History Course.
# Bob enrolled in the History Course.

from abc import ABC, abstractmethod


class Animal(ABC):
    def __init__(self, name: str) -> None:
        self.name = name

    @abstractmethod
    def sound(self) -> None:
        ...

    @abstractmethod
    def move(self) -> None:
        ...

class Cat(Animal):
    def __init__(self, name: str) -> None:
        super().__init__(name)

    def sound(self) -> None:
        print("Meow!")

    def move(self) -> None:
        print("Paw paw paw...")


cat: Cat = Cat("Tom")
cat.sound() # Meow!
cat.move() # Paw paw paw...

# bultin type constructor without parameter

print(int()) # 0
print(bool()) # False
print(repr(str())) # ''
print(list()) # []
print(set()) # set()
print(dict()) # {}
print(tuple()) # ()


from collections import defaultdict

# d: defaultdict[str, int] = defaultdict(lambda: 0)
d: defaultdict[str, int] = defaultdict(int)

print(d["a"])  # 0

# keyword and soft keyword

import keyword

print(keyword.softkwlist)
print(keyword.kwlist)

# ['_', 'case', 'match', 'type'] 4
# ['False', 'None', 'True', 'and', 'as', 'assert', 'async', 'await', 'break', 'class', 'continue', 'def', 'del', 'elif', 'else', 'except', 'finally', 'for', 'from', 'global', 'if', 'import', 'in', 'is', 'lambda', 'nonlocal', 'not', 'or', 'pass', 'raise', 'return', 'try', 'while', 'with', 'yield'] 35

# self is not keyword
self = "self"
this = self
print(this) # self

# match is soft keyword
match = "match"
switch = match
print(switch) # match

# as is keyword
as = 10  # SyntaxError: invalid syntax
print(as) # SyntaxError: invalid syntax

# class arribute and instance arribute

class Car:
    fuel_type: str = "electric"

    def __init__(self, brand: str) -> None:
        self.brand = brand


bmw: Car = Car("BMW")
print(bmw.__dict__) # {'brand': 'BMW'}
print(bmw.fuel_type) # electric  instance -> class

bmw.fuel_type = "disel" # add to instance attribute, not effect class attribute
print(Car.fuel_type) # "disel"
print(bmw.__dict__) # {'brand': 'BMW', 'fuel_type': 'disel'}

# slots

UsingSlots

Why Use __slots__ ? The short answer is slots are more efficient in terms of memory space and speed of access, and a bit safer than the default Python method of data access. By default, when Python creates a new instance of a class, it creates a __dict__ attribute for the class. The __dict__ attribute is a dictionary whose keys are the variable names and whose values are the variable values. This allows for dynamic variable creation but can also lead to uncaught errors. For example, with the default __dict__ , a misspelled variable name results in the creation of a new variable, but with __slots__ it raises in an AttributeError.

class Location:
    __slots__ = 'name', '_longitude', '_latitude'
    
    def __init__(self, name, longitude, latitude):
        self._longitude = longitude
        self._latitude = latitude
        self.name = name
        
    @property
    def longitude(self):
        return self._longitude
    
    @property
    def latitude(self):
        return self._latitude
    

print(Location.__dict__) # {'__module__': '__main__', '__firstlineno__': 383, '__slots__': ('name', '_longitude', '_latitude'), '__init__': <function Location.__init__ at 0x000001C3F4198860>, 'longitude': <property object at 0x000001C3F41F6840>, 'latitude': <property object at 0x000001C3F421F9C0>, '__static_attributes__': ('_latitude', '_longitude', 'name'), '_latitude': <member '_latitude' of 'Location' objects>, '_longitude': <member '_longitude' of 'Location' objects>, 'name': <member 'name' of 'Location' objects>, '__doc__': None}
Location.map_service = "Google Maps"
print(Location.__dict__) # {'__module__': '__main__', '__firstlineno__': 383, '__slots__': ('name', '_longitude', '_latitude'), '__init__': <function Location.__init__ at 0x000001C3F4198860>, 'longitude': <property object at 0x000001C3F41F6840>, 'latitude': <property object at 0x000001C3F421F9C0>, '__static_attributes__': ('_latitude', '_longitude', 'name'), '_latitude': <member '_latitude' of 'Location' objects>, '_longitude': <member '_longitude' of 'Location' objects>, 'name': <member 'name' of 'Location' objects>, '__doc__': None, 'map_service': 'Google Maps'}

l = Location('Mumbai', 19.0760, 72.8777)
print(l.name, l.longitude, l.latitude) # Mumbai 19.076 72.8777

try:
    l.__dict__
except AttributeError as ex:
    print(ex) # 'Location' object has no attribute '__dict__'


try:
    l.map_link = 'http://maps.google.com/...'
except AttributeError as ex:
    print(ex) # 'Location' object has no attribute 'map_link' and no __dict__ for setting new attributes

dataclass use slots

from dataclasses import dataclass

@dataclass(slots=True)
class Fruit:
    name: str
    calories: float


apple = Fruit("Apple", 20)
print(apple)
print(apple.__dict__) # AttributeError: 'Fruit' object has no attribute '__dict__'

# private attribute

from uuid import uuid4


class User:
    def __init__(self) -> None:
        self._id = uuid4()

    def _get_id(self):
        return self._id
    

user = User()
print(user._get_id(), user._id) # 940a4375-027a-4cee-9f90-2259ca27d2f5 940a4375-027a-4cee-9f90-2259ca27d2f5

class User:
    def __init__(self) -> None:
        self.__id = uuid4()

    def __get_id(self):
        return self.__id
    

user = User()
# print(user.__id) # AttributeError: 'User' object has no attribute '__id'
# print(user.__get_id()) # AttributeError: 'User' object has no attribute '__get_id'
print(user.__dict__) # {'_User__id': UUID('e02b8b86-862c-4ed0-ba1b-147f5660058e')}
print(user._User__id, user._User__get_id()) # e02b8b86-862c-4ed0-ba1b-147f5660058e e02b8b86-862c-4ed0-ba1b-147f5660058e

# typing

from __future__ import annotations  #  type checking after reading the whole file
from datetime import datetime
from typing import Callable, Literal, TypeVar, Sequence, Optional
from collections.abc import Iterable


class Student:
    def __init__(
        self,
        name: str,
        birthdate: datetime | str,
        sex: Literal["M", "F"],
        courses: list[str],
        scores: dict[str, float],
        location: tuple[float, float] | None,
    ):
        self.name = name
        self.birthdate = birthdate
        self.sex = sex
        self.courses = courses
        self.scores = scores
        self.location = location

    # def follow(self, other: "Student"):
    #     pass

    def follow(self, other: Student):  #  from __future__ import annotations
        pass


s = Student(
    name="Alice",
    birthdate="2000-01-01",
    sex="F",
    courses=["Math", "Science"],
    scores={"Math": 95.5},
    location=(40.7128, -74.0060),
)


# Be liberal in what you accept and conservative in what you send
def print_names(names: Iterable[str]) -> None:
    for name in names:
        print(name)


print_names(["Alice", "Bob", "Charlie"])
print_names(("Alice", "Bob", "Charlie"))


def add(x: int, y: int) -> int:
    return x + y


callable: Callable[[int, int], int] = add
print(callable(2, 3))


def apply_function(func: Callable[[int, int], int], x: int, y: int) -> int:
    return func(x, y)


print(apply_function(add, 5, 7))


def add_two_items[T: (int, str)](a: T, b: T) -> T:
    return a + b


U = TypeVar("U", int, str)


def add_two_items_again(a: U, b: U) -> U:
    return a + b


print(add_two_items(1, 2))  #  Outputs: 3
print(add_two_items("Hello, ", "World!"))  #  Outputs: Hello, World!
# print(add_two_items("a", 1))


Num = float | int


def add_num(a: Num, b: Num) -> Optional[Num]:
    if a <= 0 or b <= 0:
        return None
    return a + b


print(add_num(1, 2))
print(add_num(1.1, 2.2))


def print_num(n: Num) -> None:
    print(n)


result = add_num(-1, 1)

if result:
    print_num(result)


# def print_num_list(lst: list[Num]) -> None:
#     for n in lst:
#         print(n)
#     lst.append(1.1)


# int_list: list[int] = [1, 2, 3]
# print_num_list(int_list)


def print_num_list(lst: Sequence[Num]) -> None:
    for n in lst:
        print(n)
    # lst.append(1.1)


int_list: list[int] = [1, 2, 3]
print_num_list(int_list)


from typing import TypeAlias, Literal

# Mode: TypeAlias = Literal["r", "w", "a"]
type Mode = Literal["r", "w", "a"]

def open_file(file: str, mode: Mode) -> str:
    return f"Reading {file} in '{mode}' mode"

print(open_file("test.txt", "r"))
print(open_file("test.txt", "o")) # Argument of type "Literal['o']" cannot be assigned to parameter "mode" of type

Never and NoReturn

Never and NoReturn have the same meaning in the type system and static type checkers treat both equivalently.

Added in version 3.6.2: Added NoReturn .

Added in version 3.11: Added Never .

from enum import Enum
from typing import Never, NoReturn


def assert_never(arg: Never) -> NoReturn:
    raise ValueError("Expected code is unreachable")

class State(Enum):
    OFF = 0
    ON = 1
    LIMBO = 2

state: State = State.ON

if state == State.ON:
    print("Turned on")
elif state == State.OFF:
    print("Turned off")
else:
    assert_never(state)
    
# mypy ./test.py 
# test.py:20: error: Argument 1 to "assert_never" has incompatible type "Literal[State.LIMBO]"; expected "Never"  [arg-type]
# Found 1 error in 1 file (checked 1 source file)

# Dunder methods

# __get_item__

from dataclasses import dataclass


@dataclass(kw_only=True)
class Fruit:
    name: str
    grames: float


class Basket:
    def __init__(self, *, fruits: list[Fruit]) -> None:
        self.fruits = fruits

    def __getitem__(self, fruit_name: str) -> list[Fruit]:
        return [fruit for fruit in self.fruits if fruit.name.lower() == fruit_name.lower() ]



fruits: list[Fruit] = [Fruit(name="Apple", grames=2500),
                       Fruit(name="Apple", grames=1500),
                       Fruit(name="Orange", grames=2500),
                       Fruit(name="Apple", grames=2500),
                       Fruit(name="Banana", grames=500),]

basket: Basket = Basket(fruits=fruits)
apples = basket["apple"]
print(apples) # [Fruit(name='Apple', grames=2500), Fruit(name='Apple', grames=1500), Fruit(name='Apple', grames=2500)]

from functools import lru_cache


class Fib:
    def __init__(self, n):
        self.n = n

    def __len__(self):
        return self.n

    def __getitem__(self, s):
        if isinstance(s, int):
            if s < 0:
                s += self.n

            if s < 0 or s >= self.n:
                raise IndexError
            else:
                return Fib._fib(s)
        else:
            # print(s, type(s))  # slice(1, None, 2) <class 'slice'>
            rng = range(*s.indices(self.n))
            return [Fib._fib(i) for i in rng]

    @staticmethod
    @lru_cache(2**10)
    def _fib(n):
        if n < 2:
            return 1
        else:
            return Fib._fib(n - 1) + Fib._fib(n - 2)


fib = Fib(10)
print(list(fib)) # [1, 1, 2, 3, 5, 8, 13, 21, 34, 55]
print(fib[0], fib[5]) # 1 8
print(fib[::-1]) # [55, 34, 21, 13, 8, 5, 3, 2, 1, 1]
print(fib[1::2]) # [1, 3, 8, 21, 55]

# __set_item__

from collections import namedtuple

Point = namedtuple("Point", "x y")


class Polygon:
    def __init__(self, *pts):
        if pts:
            self._pts = [Point(*pt) for pt in pts]
        else:
            self._pts = []

    def __repr__(self):
        pts_str = ", ".join([str(pt) for pt in self._pts])
        return f"Polygon({pts_str})"

    def __len__(self):
        return len(self._pts)

    def __getitem__(self, s):
        return self._pts[s]

    def __setitem__(self, s, value):
        # we first should see if we have a single Point
        # or an iterable of Points in value
        try:
            rhs = [Point(*pt) for pt in value]
            is_single = False
        except TypeError:
            # not a valid iterable of Points
            # maybe a single Point?
            try:
                rhs = Point(*value)
                is_single = True
            except TypeError:
                # still no go
                raise TypeError("Invalid Point or iterable of Points")

        # reached here, so rhs is either an iterable of Points, or a Point
        # we want to make sure we are assigning to a slice only if we
        # have an iterable of points, and assigning to an index if we
        # have a single Point only
        if (isinstance(s, int) and is_single) or isinstance(s, slice) and not is_single:
            self._pts[s] = rhs
        else:
            raise TypeError("Incompatible index/slice assignment")

    def __add__(self, pt):
        if isinstance(pt, Polygon):
            new_pts = self._pts + pt._pts
            return Polygon(*new_pts)
        else:
            raise TypeError("can only concatenate with another Polygon")

    def append(self, pt):
        self._pts.append(Point(*pt))

    def extend(self, pts):
        if isinstance(pts, Polygon):
            self._pts = self._pts + pts._pts
        else:
            # assume we are being passed an iterable containing Points
            # or something compatible with Points
            points = [Point(*pt) for pt in pts]
            self._pts = self._pts + points

    def __iadd__(self, pts):
        self.extend(pts)
        return self

    def insert(self, i, pt):
        self._pts.insert(i, Point(*pt))

    def __delitem__(self, s):
        del self._pts[s]

    def pop(self, i):
        return self._pts.pop(i)


p = Polygon((0, 0), (1, 1), (2, 2))
print(p)  # Polygon(Point(x=0, y=0), Point(x=1, y=1), Point(x=2, y=2))
print(p[0])  # Point(x=0, y=0)

p[0] = (1, 1)
print(p)  # Polygon(Point(x=1, y=1), Point(x=1, y=1), Point(x=2, y=2))

p[0:2] = [(0, 0), (0, 1)]
print(p)  # Polygon(Point(x=0, y=0), Point(x=0, y=1), Point(x=2, y=2))

# iter

class Cities:
    def __init__(self):
        self._cities = ["Paris", "Berlin", "Rome", "Madrid", "London"]
        self._index = 0

    def __len__(self):
        return len(self._cities)

    def __iter__(self):
        print("Cities __iter__ called")
        return self.CityIterator(self)

    def __getitem__(self, s):
        print(f"Cities __getitem__ {s}")
        return self._cities[s]

    class CityIterator:
        def __init__(self, city_obj):
            print("CityIterator new object")
            self._city_obj = city_obj
            self._index = 0

        def __iter__(self):
            print("CityIterator __iter__ called")
            return self

        def __next__(self):
            print("CityIterator __next__ called")
            if self._index < len(self._city_obj):
                item = self._city_obj._cities[self._index]
                self._index += 1
                return item
            else:
                raise StopIteration


cities = Cities()

print(cities[0])
# Cities __getitem__ 0
# Paris

print(cities[0:4:2])
# Cities __getitem__ slice(0, 4, 2)
# ['Paris', 'Rome']


for city in cities:
    print(city)

# Cities __iter__ called
# CityIterator new object
# CityIterator __next__ called
# Paris
# CityIterator __next__ called
# Berlin
# CityIterator __next__ called
# Rome
# CityIterator __next__ called
# Madrid
# CityIterator __next__ called
# London
# CityIterator __next__ called

class Cities:
    def __init__(self):
        self._cities = ["Paris", "Berlin", "Rome", "Madrid", "London"]
        self._index = 0

    def __len__(self):
        return len(self._cities)

    def __getitem__(self, s):
        print(f"Cities __getitem__ {s}")
        return self._cities[s]


cities = Cities()

for city in cities:
    print(city)

# Cities __getitem__ 0
# Paris
# Cities __getitem__ 1
# Berlin
# Cities __getitem__ 2
# Rome
# Cities __getitem__ 3
# Madrid
# Cities __getitem__ 4
# London
# Cities __getitem__ 5

from collections import namedtuple

_SUITS = ("Spades", "Hearts", "Diamonds", "Clubs")
_RANKS = tuple(range(2, 11)) + tuple("JQKA")


Card = namedtuple("Card", "rank suit")


class CardDeck:
    def __init__(self):
        self.length = len(_SUITS) * len(_RANKS)

    def __len__(self):
        return self.length

    def __iter__(self):
        return self.CardDeckIterator(self.length)

    def __reversed__(self):
        return self.CardDeckIterator(self.length, reverse=True)

    class CardDeckIterator:
        def __init__(self, length, reverse=False):
            self.length = length
            self.i = 0
            self.reverse = reverse

        def __iter__(self):
            return self

        def __next__(self):
            if self.i < self.length:
                if self.reverse:
                    index = self.length - 1 - self.i
                else:
                    index = self.i

                suit = _SUITS[index // len(_RANKS)]
                rank = _RANKS[index % len(_RANKS)]
                self.i += 1
                return Card(rank, suit)
            else:
                raise StopIteration


deck = CardDeck()

# for card in deck:
#     print(card)
# Card(rank=2, suit="Spades")
# Card(rank=3, suit="Spades")
# Card(rank=4, suit="Spades")
# Card(rank=5, suit="Spades")
# ...

for card in reversed(deck):
    print(card)

# Card(rank="A", suit="Clubs")
# Card(rank="K", suit="Clubs")
# Card(rank="Q", suit="Clubs")
# Card(rank="J", suit="Clubs")
# ...

# format

from typing import Self


class Fruit:
    def __init__(self, *, name: str, grams: float) -> None:
        self.name = name
        self.grams = grams

    def __format__(self, format_spec: str) -> str:
        match format_spec:
            case "kg":
                return f"{self.grams / 1000:.2f}kg"
            case "lb":
                return f"{self.grams / 453.5924:.2f}lb"
            case "desc":
                return f"{self.grams}g of {self.name}"
            case _:
                raise ValueError("Unknown format specifier...")

    def __eq__(self, other: Self) -> bool:
        return self.__dict__ == other.__dict__
    
    def __or__(self, other: Self) -> Self:
        new_name: str = f"{self.name} & {other.name}"
        new_grams: float = self.grams + other.grams
        return Fruit(name=new_name, grams=new_grams)
    
    def __str__(self) -> str:
        return f"Fruit(name={self.name}, grams={self.grams})"
    
print(f"{f1:kg}") # 1.20kg
print(f"{f1:lb}") # 2.65lb
print(f"{f1:desc}") # 1200g of Apple

f1 = Fruit(name="Apple", grams=1200)
f2 = Fruit(name="Apple", grams=1200)
print(f1 == f2) # True

f3 = Fruit(name="Orange", grams=2200)
print(f1 | f3) # Fruit(name=Apple & Orange, grams=3400)

from datetime import date


class Person:
    def __init__(self, name, dob):
        self.name = name
        self.dob = dob

    def __repr__(self):
        print("__repr__ called...")
        return f"Person(name={self.name}, dob={self.dob.isoformat()})"

    def __str__(self):
        print("__str__ called...")
        return f"Person({self.name})"

    def __format__(self, date_format_spec):
        print(f"__format__ called with {repr(date_format_spec)}...")
        dob = format(self.dob, date_format_spec)
        return f"Person(name={self.name}, dob={dob})"


p = Person("Alex", date(1900, 10, 20))
print(format(p, "%B %d, %Y"))
# __format__ called with '%B %d, %Y'...
# Person(name=Alex, dob=October 20, 1900)

# sub

通过定义 __sub__ 方法可以给类型添加 - 操作

a = [1, 2, 3]
b = [2, 3, 4]
print(a + b) # [1, 2, 3, 2, 3, 4]

class SubList(list):
    def __sub__(self, b):
        a = self[:]
        b = b[:]
        while len(b) > 0:
            ele = b.pop()
            if ele in a:
                a.remove(ele)
        return a

al = SubList(a)
bl = SubList(b)
print(al - bl) # [1]

# call

import random
from time import sleep, perf_counter
from functools import wraps


def profiler(fn):
    _counter = 0
    _total_elapsed = 0
    _avg_time = 0

    @wraps(fn)
    def inner(*args, **kwargs):
        nonlocal _counter
        nonlocal _total_elapsed
        nonlocal _avg_time
        _counter += 1
        start = perf_counter()
        result = fn(*args, **kwargs)
        end = perf_counter()
        _total_elapsed += end - start
        return result

    # we need to give a way to our users to look at the
    # counter and avg_time values - but we need to make sure
    # it is using a cell reference!
    def counter():
        # this will now be a closure with a cell pointing to _counter
        return _counter

    def avg_time():
        return _total_elapsed / _counter

    inner.counter = counter
    inner.avg_time = avg_time
    return inner


@profiler
def fn(a, b):
    sleep(random.random())
    return a, b


print(fn(1, 2))  # (1, 2)
print(fn(3, 4))  # (3, 4)
print(fn.counter(), fn.avg_time())  # 2 0.717160550004337

import random
from time import sleep, perf_counter


class Profiler:
    def __init__(self, fn):
        self.counter = 0
        self.total_elapsed = 0
        self.fn = fn

    def __call__(self, *args, **kwargs):
        self.counter += 1
        start = perf_counter()
        result = self.fn(*args, **kwargs)
        end = perf_counter()
        self.total_elapsed += end - start
        return result

    @property
    def avg_time(self):
        return self.total_elapsed / self.counter


@Profiler
def fn(a, b):
    sleep(random.random())
    return a, b


print(fn(1, 2))  # (1, 2)
print(fn(3, 4))  # (3, 4)
print(fn.counter, fn.avg_time)  # 2 0.6495584000367671

# MethodType

from types import MethodType


class Person:
    def __init__(self, name):
        self.name = name


def say_hello(self):
    return f"{self.name} says hello!"


person = Person("Eric")

print(say_hello)  # <function say_hello at 0x0000021C1F548860>
print(say_hello(person))  # Eric says hello!


person_say_hello = MethodType(say_hello, person)
print(person_say_hello)
# <bound method say_hello of <__main__.Person object at 0x0000021C1F287230>>
print(person_say_hello())  # Eric says hello!
print(person.__init__)
# <bound method Person.__init__ of <__main__.Person object at 0x0000021C1F287230>>

from types import MethodType


class Person:
    def __init__(self, name):
        self.name = name

    def register_do_work(self, func):
        setattr(self, "_do_work", MethodType(func, self))

    def do_work(self):
        do_work_method = getattr(self, "_do_work", None)
        if do_work_method:
            return do_work_method()
        else:
            raise AttributeError("You must first register a do_work method")


def work_math(self):
    return f"{self.name} will teach differentials today."


math_teacher = Person("Eric")
math_teacher.register_do_work(work_math)

print(math_teacher.do_work())  # math_teacher = Person("Eric")

# meta

import math

class_name = "Circle"
class_body = """
def __init__(self, x, y, r):
    self.x = x
    self.y = y
    self.r = r

def area(self):
    return math.pi * self.r ** 2
"""
class_bases = ()  # defaults to object
class_dict = {}

exec(class_body, globals(), class_dict)

print(class_dict)
# {'__init__': <function __init__ at 0x0000025E86788860>, 'area': <function area at 0x0000025E867ED120>}

Circle = type(class_name, class_bases, class_dict)
print(Circle)  # <class '__main__.Circle'>
print(type(Circle))  # <class 'type'>
print(Circle.__dict__)
# {'__init__': <function __init__ at 0x0000025E86788860>, 'area': <function area at 0x0000025E867ED120>, '__module__': '__main__', '__dict__': <attribute '__dict__' of 'Circle' objects>, '__weakref__': <attribute '__weakref__' of 'Circle' objects>, '__doc__': None

circle = Circle(0, 0, 1)
print(circle.x, circle.y, circle.r, circle.area())  # 0 0 1 3.141592653589793

import math


class CustomType(type):
    def __new__(cls, name, bases, class_dict):
        # above is the signature that type.__new__ has -
        # and args are collected and passed by Python when we call a class (to create an instance of that class)
        # we'll see where those actually come from later
        print("Customized type creation!")
        cls_obj = super().__new__(
            cls, name, bases, class_dict
        )  # delegate to super (type in this case)
        cls_obj.circ = (
            lambda self: 2 * math.pi * self.r
        )  # basically attaching a function to the class
        return cls_obj


class_body = """
def __init__(self, x, y, r):
    self.x = x
    self.y = y
    self.r = r

def area(self):
    return math.pi * self.r ** 2
"""


class_dict = {}
exec(class_body, globals(), class_dict)
print(class_dict)
# {'__init__': <function __init__ at 0x000001FDDC57D120>, 'area': <function area at 0x000001FDDC57F7E0>}


Circle = CustomType("Circle", (), class_dict)
print(Circle)  # <class '__main__.Circle'>
print(type(Circle))  # <class '__main__.CustomType'>


c = Circle(0, 0, 1)
print(c.area(), c.circ())  # 3.141592653589793 6.283185307179586

import math


class CustomType(type):
    def __new__(cls, name, bases, class_dict):
        print(f"Using custom metaclass {cls} to create class {name}")
        cls_obj = super().__new__(cls, name, bases, class_dict)
        cls_obj.circ = lambda self: 2 * math.pi * self.r
        return cls_obj


class Circle(metaclass=CustomType):
    def __init__(self, x, y, r):
        self.x = x
        self.y = y
        self.r = r

    def area(self):
        return math.pi * self.r**2


c = Circle(0, 0, 1)
print(c.area(), c.circ())
# Using custom metaclass <class '__main__.CustomType'> to create class Circle
# 3.141592653589793 6.283185307179586

from functools import wraps


def func_logger(fn):
    @wraps(fn)
    def inner(*args, **kwargs):
        result = fn(*args, **kwargs)
        print(f"log: {fn.__qualname__}({args}, {kwargs}) = {result}")
        return result

    return inner


class Person:
    @func_logger
    def __init__(self, name, age):
        self.name = name
        self.age = age

    @func_logger
    def greet(self):
        return f"Hello, my name is {self.name} and I am {self.age}"


p = Person("John", 78)
print(p.greet())
# log: Person.__init__((<__main__.Person object at 0x00000231640F7230>, 'John', 78), {}) = None
# log: Person.greet((<__main__.Person object at 0x00000231640F7230>,), {}) = Hello, my name is John and I am 78
# Hello, my name is John and I am 78


def class_logger(cls):
    for name, obj in vars(cls).items():
        if callable(obj):
            print("decorating:", cls, name)
            setattr(cls, name, func_logger(obj))
    return cls


@class_logger
class Person:
    def __init__(self, name, age):
        self.name = name
        self.age = age

    def greet(self):
        return f"Hello, my name is {self.name} and I am {self.age}"


# decorating: <class '__main__.Person'> __init__
# decorating: <class '__main__.Person'> greet

p = Person("John", 78)
print(p.greet())
# log: Person.__init__((<__main__.Person object at 0x000001CC950F74D0>, 'John', 78), {}) = None
# log: Person.greet((<__main__.Person object at 0x000001CC950F74D0>,), {}) = Hello, my name is John and I am 78
# Hello, my name is John and I am 78

import inspect
from functools import wraps


def func_logger(fn):
    @wraps(fn)
    def inner(*args, **kwargs):
        result = fn(*args, **kwargs)
        print(f"log: {fn.__qualname__}({args}, {kwargs}) = {result}")
        return result

    return inner


def class_logger(cls):
    for name, obj in vars(cls).items():
        if isinstance(obj, staticmethod):
            original_func = obj.__func__
            print("decorating static method", original_func)
            decorated_func = func_logger(original_func)
            method = staticmethod(decorated_func)
            setattr(cls, name, method)
        elif isinstance(obj, classmethod):
            original_func = obj.__func__
            print("decorating class method", original_func)
            decorated_func = func_logger(original_func)
            method = classmethod(decorated_func)
            setattr(cls, name, method)
        elif isinstance(obj, property):
            print("decorating property", obj)
            if obj.fget:
                obj = obj.getter(func_logger(obj.fget))
            if obj.fset:
                obj = obj.setter(func_logger(obj.fset))
            if obj.fdel:
                obj = obj.deleter(func_logger(obj.fdel))
            setattr(cls, name, obj)
        elif inspect.isroutine(obj):
            print("decorating:", cls, name)
            setattr(cls, name, func_logger(obj))
    return cls


@class_logger
class MyClass:
    def __init__(self):
        print("__init__ called...")

    @staticmethod
    def static_method():
        print("static method called...")

    @classmethod
    def cls_method(cls):
        print("class method called...")

    def inst_method(self):
        print("instance method called...")

    @property
    def name(self):
        print("name getter called...")

    def __add__(self, other):
        print("__add__ called...")

    @class_logger
    class Other:
        def __call__(self):
            print(f"{self}.__call__ called...")

    other = Other()


# decorating: <class '__main__.MyClass.Other'> __call__
# decorating static method <function MyClass.static_method at 0x000001511249F060>
# decorating class method <function MyClass.cls_method at 0x000001511249F1A0>
# decorating: <class '__main__.MyClass'> inst_method
# decorating property <property object at 0x00000151120C6660>
# decorating: <class '__main__.MyClass'> __add__

my_class = MyClass()
# __init__ called...
# log: MyClass.__init__((<__main__.MyClass object at 0x00000226EBCC8830>,), {}) = None

my_class.inst_method()
# instance method called...
# log: MyClass.inst_method((<__main__.MyClass object at 0x00000226EBCC8830>,), {}) = None


MyClass.static_method()
# static method called...
# log: MyClass.static_method((), {}) = None

class Logger:
    def __init__(self, fn):
        self.fn = fn

    def __call__(self, *args, **kwds):
        print(f"Log: {self.fn.__name__} called.")
        return self.fn(*args, **kwds)


class Person:
    def __init__(self, name):
        self.name = name

    @Logger
    def say_hello(self):
        return f"{self.name} says hello!"


p = Person("David")
# p.say_hello()  # TypeError: Person.say_hello() missing 1 required positional argument: 'self'
print(p.say_hello)  # <__main__.Logger object at 0x000001FFF8F77230>



from types import MethodType


class Logger:
    def __init__(self, fn):
        self.fn = fn

    def __call__(self, *args, **kwargs):
        print(f"Log: {self.fn.__name__} called.")
        return self.fn(*args, **kwargs)

    def __get__(self, instance, owner_class):
        print(f"__get__ called: self={self}, instance={instance}")
        if instance is None:
            print("\treturning self unbound...")
            return self
        else:
            # self is callable, since it implements __call__
            print("\treturning self as a method bound to instance")
            return MethodType(self, instance)


class Person:
    def __init__(self, name):
        self.name = name

    @Logger
    def say_hello(self):
        return f"{self.name} says hello!"


p = Person("David")
print(p.say_hello)
# __get__ called: self=<__main__.Logger object at 0x000002050BE774D0>, instance=<__main__.Person object at 0x000002050BE77620>
#         returning self as a method bound to instance
# <bound method ? of <__main__.Person object at 0x000002050BE77620>>
print(p.say_hello())
# __get__ called: self=<__main__.Logger object at 0x000001D1E5B174D0>, instance=<__main__.Person object at 0x000001D1E5B17620>
#         returning self as a method bound to instance
# Log: say_hello called.
# David says hello!


@Logger
def say_bye():
    pass


say_bye()  # Log: say_bye called.


class Person:
    @classmethod
    @Logger
    def cls_method(cls):
        print("class method called...")

    @staticmethod
    @Logger
    def static_method():
        print("static method called...")

    @Logger
    def cls_method2():
        print("class method called...")


Person.cls_method()
# Log: cls_method called.
# class method called...

Person.static_method()
# Log: static_method called.
# static method called...

Person.cls_method2()
# __get__ called: self=<__main__.Logger object at 0x000001C340A8B820>, instance=None
#         returning self unbound...
# Log: cls_method2 called.
# class method called...

from functools import wraps


def func_logger(fn):
    @wraps(fn)
    def inner(*args, **kwargs):
        result = fn(*args, **kwargs)
        print(f"log: {fn.__qualname__}({args}, {kwargs}) = {result}")
        return result

    return inner


def class_logger(cls):
    for name, obj in vars(cls).items():
        if callable(obj):
            print("decorating:", cls, name)
            setattr(cls, name, func_logger(obj))
    return cls


@class_logger
class Person:
    def __init__(self, name, age):
        self.name = name
        self.age = age

    def greet(self):
        return f"Hello, my name is {self.name} and I am {self.age}"


# decorating: <class '__main__.Person'> __init__
# decorating: <class '__main__.Person'> greet

Person("Alex", 10).greet()
# log: Person.__init__((<__main__.Person object at 0x0000025672C47230>, 'Alex', 10), {}) = None
# log: Person.greet((<__main__.Person object at 0x0000025672C47230>,), {}) = Hello, my name is Alex and I am 10


class Student(Person):
    def __init__(self, name, age, student_number):
        super().__init__(name, age)
        self.student_number = student_number

    def study(self):
        return f"{self.name} studies..."


Student("Alex", 19, "001").study()
# log: Person.__init__((<__main__.Student object at 0x0000025660E47230>, 'Alex', 19), {}) = None


class ClassLogger(type):
    def __new__(mcls, name, bases, class_dict):
        new_cls = super().__new__(mcls, name, bases, class_dict)
        for key, obj in vars(new_cls).items():
            if callable(obj):
                setattr(new_cls, key, func_logger(obj))
        return new_cls


class Person(metaclass=ClassLogger):
    def __init__(self, name, age):
        self.name = name
        self.age = age

    def greet(self):
        return f"Hello, my name is {self.name} and I am {self.age}"


Person("John", 78).greet()
# log: Person.__init__((<__main__.Person object at 0x0000015640237230>, 'John', 78), {}) = None
# log: Person.greet((<__main__.Person object at 0x0000015640237230>,), {}) = Hello, my name is John and I am 78


class Student(Person):
    def __init__(self, name, age, student_number):
        super().__init__(name, age)
        self.student_number = student_number

    def study(self):
        return f"{self.name} studies..."


Student("Alex", 19, "001").study()
# log: Person.__init__((<__main__.Student object at 0x000001A02F6B7230>, 'Alex', 19), {}) = None
# log: Student.__init__((<__main__.Student object at 0x000001A02F6B7230>, 'Alex', 19, '001'), {}) = None
# log: Student.study((<__main__.Student object at 0x000001A02F6B7230>,), {}) = Alex studies...

class AutoClassAttrib(type):
    def __new__(cls, name, bases, cls_dict, extra_attrs=None):
        new_cls = super().__new__(cls, name, bases, cls_dict)
        if extra_attrs:
            print("Creating class with some extra attributes: ", extra_attrs)
            for attr_name, attr_value in extra_attrs:
                setattr(new_cls, attr_name, attr_value)
        return new_cls


class Account(
    metaclass=AutoClassAttrib, extra_attrs=[("account_type", "Savings"), ("apr", 0.5)]
):
    pass


# Creating class with some extra attributes:  [('account_type', 'Savings'), ('apr', 0.5)]

print(vars(Account))
# {'__module__': '__main__', '__firstlineno__': 11, '__static_attributes__': (), '__dict__': <attribute '__dict__' of 'Account' objects>, '__weakref__': <attribute '__weakref__' of 'Account' objects>, '__doc__': None, 'account_type': 'Savings', 'apr': 0.5}


class AutoClassAttrib(type):
    def __new__(cls, name, bases, cls_dict, **kwargs):
        new_cls = super().__new__(cls, name, bases, cls_dict)
        if kwargs:
            print("Creating class with some extra attributes: ", kwargs)
            for attr_name, attr_value in kwargs.items():
                setattr(new_cls, attr_name, attr_value)
        return new_cls


class Account(metaclass=AutoClassAttrib, account_type="Savings", apr=0.5):
    pass


# Creating class with some extra attributes:  {'account_type': 'Savings', 'apr': 0.5}

print(vars(Account))
# {'__module__': '__main__', '__firstlineno__': 33, '__static_attributes__': (), '__dict__': <attribute '__dict__' of 'Account' objects>, '__weakref__': <attribute '__weakref__' of 'Account' objects>, '__doc__': None, 'account_type': 'Savings', 'apr': 0.5}

class MyMeta(type):
    def __new__(mcls, name, bases, cls_dict, **kwargs):
        print("MyMeta.__new__ called...")
        print("\tcls: ", mcls, type(mcls))
        print("\tname:", name, type(name))
        print("\tbases: ", bases, type(bases))
        print("\tcls_dict:", cls_dict, type(cls_dict))
        print("\tkwargs:", kwargs)
        return super().__new__(mcls, name, bases, cls_dict)


class MyClass(metaclass=MyMeta):
    pass


# MyMeta.__new__ called...
#         cls:  <class '__main__.MyMeta'> <class 'type'>
#         name: MyClass <class 'str'>
#         bases:  () <class 'tuple'>
#         cls_dict: {'__module__': '__main__', '__qualname__': 'MyClass', '__firstlineno__': 12, '__static_attributes__': ()} <class 'dict'>
#         kwargs: {}


class MyMeta(type):
    @staticmethod
    def __prepare__(name, bases, **kwargs):
        print("MyMeta.__prepare__ called...")
        print("\tname:", name)
        print("\tkwargs:", kwargs)
        kwargs["kw1"] = 100
        kwargs["kw3"] = 300
        return kwargs

    def __new__(mcls, name, bases, cls_dict, **kwargs):
        print("MyMeta.__new__ called...")
        print("\tcls: ", mcls, type(mcls))
        print("\tname:", name, type(name))
        print("\tbases: ", bases, type(bases))
        print("\tcls_dict:", cls_dict, type(cls_dict))
        print("\tkwargs:", kwargs)
        return super().__new__(mcls, name, bases, cls_dict)


class MyClass(metaclass=MyMeta, kw1=10, kw2=20):
    pass


# MyMeta.__prepare__ called...
#         name: MyClass
#         kwargs: {'kw1': 10, 'kw2': 20}
# MyMeta.__new__ called...
#         cls:  <class '__main__.MyMeta'> <class 'type'>
#         name: MyClass <class 'str'>
#         bases:  () <class 'tuple'>
#         cls_dict: {'kw1': 100, 'kw2': 20, 'kw3': 300, '__module__': '__main__', '__qualname__': 'MyClass', '__firstlineno__': 44, '__static_attributes__': ()} <class 'dict'>
#         kwargs: {'kw1': 10, 'kw2': 20}


print(vars(MyClass))
# {'kw1': 100, 'kw2': 20, 'kw3': 300, '__module__': '__main__', '__firstlineno__': 44, '__static_attributes__': (), '__dict__': <attribute '__dict__' of 'MyClass' objects>, '__weakref__': <attribute '__weakref__' of 'MyClass' objects>, '__doc__': None}


my_class = MyClass()
print(my_class.kw1, my_class.kw2, my_class.kw3)  # 100 20 300


class CustomDict(dict):
    def __setitem__(self, key, value):
        print(f"Setting {key} = {value} in custom dictionary")
        super().__setitem__(key, value)

    def __getitem__(self, key):
        print(f"Getting {key} from custom dictionary")
        return int(super().__getitem__(key))


class MyMeta(type):
    def __prepare__(name, bases):
        return CustomDict()

    def __new__(mcls, name, bases, cls_dict):
        print("metaclass __new__ called...")
        print(f"\ttype(cls_dict) = {type(cls_dict)}")
        print(f"\tcls_dict={cls_dict}")


class MyClass(metaclass=MyMeta):
    pass


# Getting __name__ from custom dictionary
# Setting __module__ = __main__ in custom dictionary
# Setting __qualname__ = MyClass in custom dictionary
# Setting __firstlineno__ = 87 in custom dictionary
# Setting __static_attributes__ = () in custom dictionary
# metaclass __new__ called...
#         type(cls_dict) = <class '__main__.CustomDict'>
#         cls_dict={'__module__': '__main__', '__qualname__': 'MyClass', '__firstlineno__': 87, '__static_attributes__': ()}

# Default Attribute Lookup

@startuml Python Default Attribute Lookup Activity Diagram(Flow Chart)

' !theme sketchy
' !theme aws-orange
!theme materia

start

:obj.attr;
:obj.~__getattribute__("attr")\ncan override;
if (in class or parents' class dict?) then (yes)
  if (data descriptor?) then (yes)
    :~__get__;
  else (no)
    if (in instance dict?) then (yes)
      :return it;
    else (no)
      if (non-data descriptor?) then (yes)
        :~__get__;
      else (no)
        :return class attr;
      endif
    endif
  endif
else (no)
  if (in instance dict?) then (yes)
    :return it;
  else (no)
    if (override ~__getattr__) then (yes)
      :return ~__getattr__ result;
    else (no)
      :AttributeError;
    endif
  endif
endif

stop
@enduml

class Person:
    pass


p = Person()

try:
    print(p.name)
except AttributeError as ex:
    print("AttributeError", ex)
# AttributeError 'Person' object has no attribute 'name'


class Person:
    def __getattr__(self, name):
        print(f"__getattribute__ did not find {name}")
        return "not found"


p = Person()
print(p.name)
# __getattribute__ did not find name
# not found


class Person:
    def __init__(self, age):
        self._age = age

    def __getattr__(self, name):
        print(f"Could not find {name}")
        alt_name = "_" + name
        try:
            return super().__getattribute__(alt_name)
        except AttributeError:
            raise AttributeError(f"Could not find {name} or {alt_name}")


p = Person(10)
print(p.age)
# Could not find age
# 10


class DefaultClass:
    def __init__(self, attribute_default=None):
        self._attribute_default = attribute_default

    def __getattr__(self, name):
        print(f"{name} not found. creating it and setting it to default...")
        setattr(self, name, self._attribute_default)
        return self._attribute_default


d = DefaultClass("NotAvailable")
print(d.name)
# name not found. creating it and setting it to default...
# NotAvailable


class Person(DefaultClass):
    def __init__(self, name):
        super().__init__("Unavailable")
        self.name = name


p = Person("Raymond")
print(p.name)
print(p.age)
# age not found. creating it and setting it to default...
# Unavailable


class AttributeNotFoundLogger:
    def __getattr__(self, name):
        err_msg = f"'{type(self).__name__}' object has no attribute '{name}'"
        print(f"Log: {err_msg}")
        raise AttributeError(err_msg)


class Person(AttributeNotFoundLogger):
    def __init__(self, name):
        self.name = name


p = Person("Raymond")
try:
    p.age
except AttributeError as ex:
    print(f"AttributeError raised: {ex}")
# Log: 'Person' object has no attribute 'age'
# AttributeError raised: 'Person' object has no attribute 'age'

class Person:
    def __init__(self, name, age):
        self._name = name
        self._age = age

    def __getattribute__(self, name: str):
        if name.startswith("_"):
            raise AttributeError(f"Forbidde access to {name}")
        return super().__getattribute__(name)


p = Person("Alex", 19)
try:
    print(p._name)
except AttributeError as ex:
    print(ex)
# Forbidde access to _name


# print(p.__dict__["_name"])
# AttributeError: Forbidde access to __dict__


class Person:
    def __init__(self, name, age):
        self._name = name
        self._age = age

    def __getattribute__(self, name: str):
        if name.startswith("_") and not name.startswith("__"):
            raise AttributeError(f"Forbidde access to {name}")
        return super().__getattribute__(name)


p = Person("Eric", 78)
print(p.__dict__)  # {'_name': 'Eric', '_age': 78}


class Person:
    def __init__(self, name, age):
        self._name = name
        self._age = age

    def __getattribute__(self, name):
        if name.startswith("_") and not name.startswith("__"):
            raise AttributeError(f"Forbidden access to {name}")
        return super().__getattribute__(name)

    @property
    def name(self):
        return self._name

    @property
    def age(self):
        return self._age


p = Person("Babb", 42)

try:
    p.name
except AttributeError as ex:
    print(ex)
# Forbidden access to _name


class Person:
    def __init__(self, name, age):
        self._name = name
        self._age = age

    def __getattribute__(self, name):
        if name.startswith("_") and not name.startswith("__"):
            raise AttributeError(f"Forbidden access to {name}")
        return super().__getattribute__(name)

    @property
    def name(self):
        return super().__getattribute__("_name")

    @property
    def age(self):
        return super().__getattribute__("_age")


p = Person("Babb", 42)
print(p.name, p.age)  # Babb 42


class MetaLogger(type):
    def __getattribute__(self, name):
        print("class __getattribute__ called...")
        return super().__getattribute__(name)

    def __getattr__(self, name):
        print("class __getattr__ called...")
        return "Not Found"


class Account(metaclass=MetaLogger):
    apr = 10


print(Account.apr)
# class __getattribute__ called...
# 10


print(Account.apy)
# class __getattribute__ called...
# class __getattr__ called...
# Not Found


class MyClass:
    def __getattribute__(self, name):
        print(f"__getattribute__ called... for {name}")
        return super().__getattribute__(name)

    def __getattr__(self, name):
        print(f"__getattr__ called... for {name}")
        raise AttributeError(f"{name} not found")

    def say_hello(self):
        return "hello"


m = MyClass()
print(m.say_hello())
# __getattribute__ called... for say_hello
# hello


# m.other()  # AttributeError: other not found

# Attribute Set

class Person:
    def __setattr__(self, name, value) -> None:
        print("setting instance attribute...")
        super().__setattr__(name, value)


p = Person()
p.name = "Babb"  # setting instance attribute...

Person.class_attr = "test"


class MyMeta(type):
    def __setattr__(self, name, value) -> None:
        print("setting class attribute...")
        return super().__setattr__(name, value)


class Person(metaclass=MyMeta):
    def __setattr__(self, name, value) -> None:
        print("setting instance attribute...")
        super().__setattr__(name, value)


Person.class_attr = "test"  # setting class attribute...


class MyNonDataDesc:
    def __get__(self, instance, owner_class):
        print("__get__ called on non-data descriptor...")


class MyDataDesc:
    def __set__(self, instance, value):
        print("__set__ called on data descriptor...")

    def __get__(self, instance, owner_class):
        print("__get__ called on data descriptor...")


class MyClass:
    non_data_desc = MyNonDataDesc()
    data_desc = MyDataDesc()

    def __setattr__(self, name, value):
        print("__setattr__ called...")
        super().__setattr__(name, value)


m = MyClass()
print(m.__dict__)  # {}

m.data_desc = 100
# __setattr__ called...
# __set__ called on data descriptor...

m.non_data_desc = 200
# __setattr__ called...

print(m.__dict__)  # {'non_data_desc': 200}

m.data_desc  # __get__ called on data descriptor...
m.non_data_desc

class MyClass:
    def __setattr__(self, name, value):
        print("__setattr__ called...")
        if name.startswith("_") and not name.startswith("__"):
            raise AttributeError("Sorry, this attribute is read-only.")
        setattr(self, name, value)


m = MyClass()

try:
    m._test = "test"
except AttributeError as ex:
    print(ex)
# __setattr__ called...
# Sorry, this attribute is read-only.

# setattr will call MyClass __setattr__
# try:
#     m.test = "test"
# except RecursionError as ex:
#     print(ex)
# maximum recursion depth exceeded


class MyClass:
    def __setattr__(self, name, value):
        print("__setattr__ called...")
        if name.startswith("_") and not name.startswith("__"):
            raise AttributeError("Sorry, this attribute is read-only.")
        super().__setattr__(name, value)


m = MyClass()
m.test = "test"
print(m.__dict__)  # {'test': 'test'}

# Fun

# nonlocal

# outside function
def outer():
    message = 'local'

    # nested function
    def inner():

        message = 'nonlocal'
        print("inner:", message)

    inner()
    print("outer:", message)

outer()

# inner: nonlocal
# outer: local



# outside function
def outer():
    message = 'local'

    # nested function
    def inner():

        # declare nonlocal variable, use outer message
        nonlocal message

        message = 'nonlocal'
        print("inner:", message)

    inner()
    print("outer:", message)

outer()

# inner: nonlocal
# outer: nonlocal

x = 100 # 0


def outer_func():
    x = "hello" # 1

    def inner_func1():
        nonlocal x
        x = "python" # change 1

        def inner_func2():
            global x
            x = "python!!!" # change 0

        print("inner(before)", x)
        inner_func2()
        print("inner(after)", x)

    inner_func1()

    print("outer_func", x)


outer_func()

print(x)

# inner(before) python
# inner(after) python
# outer_func python
# python!!!

# imp

如果你的模块不在 sys.path，可以使用 imp 模块中的方法 imp.load_source.

import imp

imp.load_source("hi", "C://data/hi.py")
import hi

# 可以自己指定模块的名字，相当于 import hi as hello
imp.load_source("hello", "C://data/hi.py")
import hello

Replace imp.load_source() in python 3.12.

import importlib.util
import importlib.machinery

def load_source(modname, filename):
    loader = importlib.machinery.SourceFileLoader(modname, filename)
    spec = importlib.util.spec_from_file_location(modname, filename, loader=loader)
    module = importlib.util.module_from_spec(spec)
    # The module is always executed and not cached in sys.modules.
    # Uncomment the following line to cache the module.
    # sys.modules[module.__name__] = module
    loader.exec_module(module)
    return module

# fun code

函数是一个对象，对象有自己的作用域，即命名空间，每个命名空间对应一个 __code__ ，包含对象的相关信息。

def fun(a, b, c, *d, e, f):
    localvar = 1

print(dir(fun.__code__))

print('co_argcount:' ,fun.__code__.co_argcount)                  #位置参数个数(a,b,c)
print('co_kwonlyargcount:',fun.__code__.co_kwonlyargcount)       #仅限关键字参数个数(e,f)
print('all_local:',fun.__code__.co_nlocals)                      #函数中局部参数(位置参数+关键字+可变参数(*d) + 本地变量)
print('consts:',fun.__code__.co_consts)                          #函数内部的常量
print('filename:',fun.__code__.co_filename)                      #函数所属的文件
print('name:',fun.__code__.co_name)                              #函数名
print('func_lineno:',fun.__code__.co_firstlineno)                #函数位於第几行

# ['__class__', '__delattr__', '__dir__', '__doc__', '__eq__', '__format__', '__ge__', '__getattribute__', '__gt__', '__hash__', '__init__', '__init_subclass__', '__le__', '__lt__', '__ne__', '__new__', '__reduce__', '__reduce_ex__', '__repr__', '__setattr__', '__sizeof__', '__str__', '__subclasshook__', 'co_argcount', 'co_cellvars', 'co_code', 'co_consts', 'co_filename', 'co_firstlineno', 'co_flags', 'co_freevars', 'co_kwonlyargcount', 'co_lnotab', 'co_name', 'co_names', 'co_nlocals', 'co_stacksize', 'co_varnames']
# co_argcount: 3
# co_kwonlyargcount: 2
# all_local: 7
# consts: (None, 1)
# filename: code_test.py
# name: fun
# func_lineno: 7

# freevars

def make_averager():
    series = []

    def averager(new_value):
        series.append(new_value)
        total = sum(series)
        return total / len(series)

    return averager

avg = make_averager()
print(avg.__code__.co_freevars)
print(avg.__code__.co_varnames)

print(avg.__closure__)
print(avg.__closure__[0].cell_contents)
avg(10)
avg(11)
print(avg.__closure__[0].cell_contents)

# ('series',)
# ('new_value', 'total')
# (<cell at 0x000001C112507618: list object at 0x000001C114148E08>,)
# []
# [10, 11]


def make_averager_nonlocal():

    count = 0
    sum = 0

    def averager(new_value):
        nonlocal count
        nonlocal sum

        count += 1
        sum += new_value
        return sum / count

    return averager

print(make_averager_nonlocal().__code__.co_freevars)
# ('count', 'sum')

# closure

A Closure is a function object that remembers values in enclosing scopes even if they are not present in memory.

number = 100
lst = []
for number in range(5):
    lst.append(number)
print(number)  # 4


number = 100
lst = [number**2 for number in range(5)]
print(number)  # 100


number = 100
[number * i for i in range(5)]
print("i" in globals())  # False


fns = []
for i in range(4):
    fns.append(lambda x: x**i)

print("i" in globals(), i)  # True 3
print(fns[0](10), fns[1](10))  # 1000


i = 10
print(fns[0](10), fns[1](10))  # 10000000000 10000000000


fns = []


def func(x):
    return x**i


for i in range(4):
    fns.append(func)


print(fns[0](10), fns[1](10))  # 1000 1000
# This is closure, remembers values i in enclosing scopes (global)
print(fns[0].__closure__, fns[0].__code__.co_freevars)  # None ()


def fns():
    fns = []
    for i in range(4):
        fns.append(lambda x: x**i)
    return fns


fns = fns()
print(fns[0](10), fns[1](10))  # 1000 1000
print(
    fns[0].__closure__, fns[0].__code__.co_freevars
)  # (<cell at 0x00000227A131F580: int object at 0x00007FFA9B51E3E8>,) ('i',)


fns = [lambda x: x**i for i in range(4)] # equal def fns()
print(fns[0](10), fns[1](10))  # 1000 1000
print(
    fns[0].__closure__, fns[0].__code__.co_freevars
)  # (<cell at 0x000002217DCFA260: int object at 0x00007FFA9B51E3E8>,) ('i',)

from datetime import datetime


def log(msg, current=datetime.now()):
    print(msg, current)


log("a")  # a 2025-12-24 13:44:35.830719
log("b")  # b 2025-12-24 13:44:35.830719


fns = [lambda x, p=i: x**p for i in range(4)]
print(fns[0](10), fns[1](10))  # 1 10
print(fns[0].__closure__, fns[0].__code__.co_freevars)  # None ()

# switch case

def foo(var):
    return {
        'a': 1,
        'b': 2,
        'c': 3,
    }.get(var, 0)

print(foo('d')) # 0

def foo(var, x):
    return {
        'a': lambda x: x+1,
        'b': lambda x: str(x+2)+var,
        'c': lambda x: x+3,
    }[var](x)

print(foo('b', 2)) # 4b

def dow_switch_dict(dow):
    dow_dict = {
        1: lambda: print("Monday"),
        2: lambda: print("Tuesday"),
        3: lambda: print("Wednesday"),
        4: lambda: print("Thursday"),
        5: lambda: print("Friday"),
        6: lambda: print("Saturday"),
        7: lambda: print("Sunday"),
        "default": lambda: print("Invalid day of week"),
    }

    return dow_dict.get(dow, dow_dict["default"])()


dow_switch_dict(1)  # Monday
dow_switch_dict(100)  # Invalid day of week

def switcher(fn):
    registry = dict()
    registry["default"] = fn

    def register(case):
        def inner(fn):
            registry[case] = fn
            return fn

        return inner

    def decorator(case):
        fn = registry.get(case, registry["default"])
        return fn()

    decorator.register = register

    return decorator


@switcher
def dow():
    print("Invalid day of week")


@dow.register(1)
def _():
    print("Monday")


dow.register(2)(lambda: print("Tuesday"))
dow.register(3)(lambda: print("Wednesday"))
dow.register(4)(lambda: print("Thursday"))
dow.register(5)(lambda: print("Friday"))
dow.register(6)(lambda: print("Saturday"))
dow.register(7)(lambda: print("Sunday"))


dow(1)  # Monday
dow(2)  # Tuesday
dow(100)  # Invalid day of week

# insepect

打印源码，获得定义模块。

import inspect

print(inspect.getsource(inspect.getsource))
print(inspect.getmodule(inspect.getmodule))
print(inspect.currentframe().f_lineno)
👍
# def getsource(object):
#     """Return the text of the source code for an object.

#     The argument may be a module, class, method, function, traceback, frame,
#     or code object.  The source code is returned as a single string.  An
#     OSError is raised if the source code cannot be retrieved."""
#     lines, lnum = getsourcelines(object)
#     return ''.join(lines)

# <module 'inspect' from 'C:\\Program Files\\python36\\lib\\inspect.py'>
# 23

import inspect


def func():
    pass


class MyClass:
    def func(self):
        pass


obj = MyClass()

print(
    inspect.isfunction(obj.func),
    inspect.ismethod(obj.func),
    inspect.isroutine(obj.func),
    inspect.isfunction(func),
    inspect.ismethod(func),
    inspect.isroutine(func),
)

# False True True True False True

import inspect
from functools import wraps


class MyClass:
    @staticmethod
    def static_method():
        pass

    @classmethod
    def cls_method(cls):
        pass

    def inst_method(self):
        pass

    @property
    def name(self):
        pass

    def __add__(self, other):
        pass

    class Other:
        def __call__(self):
            pass

    other = Other()


keys = (
    "static_method",
    "cls_method",
    "inst_method",
    "name",
    "__add__",
    "Other",
    "other",
)
inspect_funcs = (
    "isroutine",
    "ismethod",
    "isfunction",
    "isbuiltin",
    "ismethoddescriptor",
)


max_header_length = max(len(key) for key in keys)
max_fname_length = max(len(func) for func in inspect_funcs)
print(
    format("", f"{max_fname_length}s"),
    "\t".join(format(key, f"{max_header_length}s") for key in keys),
)
for inspect_func in inspect_funcs:
    fn = getattr(inspect, inspect_func)
    inspect_results = (
        format(str(fn(MyClass.__dict__[key])), f"{max_header_length}s") for key in keys
    )
    print(format(inspect_func, f"{max_fname_length}s"), "\t".join(inspect_results))

#                    static_method        cls_method      inst_method     name            __add__         Other           other
# isroutine          True                 True            True            False           True            False           False
# ismethod           False                False           False           False           False           False           False
# isfunction         False                False           True            False           True            False           False
# isbuiltin          False                False           False           False           False           False           False
# ismethoddescriptor True                 True            False           False           False           False           False

print(inspect.ismethod(MyClass.inst_method))  # False
print(inspect.ismethod(MyClass().inst_method))  # True

import inspect


def func(
    a: str,
    b: "int > 0" = 1,
    *args: "some extra positioal args",
    k1: "keyword-only arg 1",
    k2: "keyword-only arg 2",
    **kwargs: "some extra keyword-only args",
) -> "return None":
    print(a, b, args, k1, k2, kwargs)


for param in inspect.signature(func).parameters.values():
    print("Name:", param.name, end=", ")
    print("Default:", param.default, end=", ")
    print("Annotation:", param.annotation, end=", ")
    print("Kind:", param.kind)
    
# Name: a, Default: <class 'inspect._empty'>, Annotation: <class 'str'>, Kind: POSITIONAL_OR_KEYWORD
# Name: b, Default: 1, Annotation: int > 0, Kind: POSITIONAL_OR_KEYWORD
# Name: args, Default: <class 'inspect._empty'>, Annotation: some extra positioal args, Kind: VAR_POSITIONAL
# Name: k1, Default: <class 'inspect._empty'>, Annotation: keyword-only arg 1, Kind: KEYWORD_ONLY
# Name: k2, Default: <class 'inspect._empty'>, Annotation: keyword-only arg 2, Kind: KEYWORD_ONLY
# Name: kwargs, Default: <class 'inspect._empty'>, Annotation: some extra keyword-only args, Kind: VAR_KEYWORD

# compile to pyc

# 单个文件
# python -m foo.py
import py_compile
py_compile.compile(file)

# 多个文件
# python -m compileall <dir>
import compileall
compileall.compile_dir(dir)

# traceback

traceback.print_exc() 可以打印出错误堆栈信息，可以显示错误行号。

try:
    1 / 0
except Exception as e:
    print(e)
# division by zero

import traceback
try:
    1 / 0
except Exception as e:
    # traceback.print_exc() <=>  print(traceback.format_exc())
    print(traceback.format_exc())
# Traceback (most recent call last):
#   File "C:/Users/11435/Desktop/Test/traceback_test.py", line 8, in <module>
#     1 / 0
# ZeroDivisionError: division by zero

# decorator

wraps(func) 可以保持原函数的状态，如果没有添加此段，返回的是包装函数的状态， whatIsLiving.__doc__ 返回 None, whatIsLiving.__name__ 返回 living

from functools import wraps

def thisIsLiving(func):
    @wraps(func)
    def living(*args,**kwargs):
        return func(*args,**kwargs) + "living is sleep and eat"
    return living

@thisIsLiving
def whatIsLiving():
    '''
    什么是活着
    '''
    return "what is living? "

print(whatIsLiving.__doc__)
print(whatIsLiving.__name__)
print(whatIsLiving())

#     什么是活着

# whatIsLiving
# what is living? living is sleep and eat

import functools
import time


def decorator(func):
    def wrapper(*args, **kwargs):
        start_time = time.time()
        result = func(*args, **kwargs)
        end_time = time.time()
        print(f"{func.__name__} execution time: {end_time - start_time}")
        return result

    return wrapper


@decorator
def square(x):
    return x * x


# square = decorator(square)
# print(square(2))

print(square(2))


def timer(threshold):
    def decorator(func):
        @functools.wraps(func)
        def wrapper(*args, **kwargs):
            start_time = time.time()

            result = func(*args, **kwargs)
            end_time = time.time()
            if end_time - start_time > threshold:
                print(
                    f"{func.__name__} execution time {end_time - start_time} took long than {threshold}"
                )
            return result

        return wrapper

    return decorator


@timer(1)
def sleep_2seconds():
    time.sleep(2)


sleep_2seconds()
print(sleep_2seconds.__name__)

decorator_factory with parameter and return decorator

def decorator_factory(a, b):
    print("running decorator_factory")

    def decorator(fn):
        print("running decorator")

        def inner(*args, **kwargs):
            print("running inner")
            print(f"{a=}, {b=}")
            return fn(*args, **kwargs)

        return inner

    return decorator


@decorator_factory(10, 20)
def func():
    print("running func")


func()

# running decorator_factory
# running decorator
# running inner
# a=10, b=20
# running func

class decorator

class MyClass:
    def __init__(self, a, b):
        self.a = a
        self.b = b

    def __call__(self, fn):
        def inner(*args, **kwargs):
            print("decorated function called: a={0}, b={1}".format(self.a, self.b))
            return fn(*args, **kwargs)

        return inner


@MyClass(10, 20)
def my_func(s):
    print("hello {0}".format(s))


my_func("World")
# decorated function called: a=10, b=20
# hello World

decorate class

import math


def complete_ordering(cls):
    # print(cls)  # <class '__main__.Point'>
    if "__eq__" in dir(cls) and "__lt__" in dir(cls):
        cls.__le__ = lambda self, other: self < other or self == other
        cls.__gt__ = lambda self, other: not (self < other) and not (self == other)
        cls.__ge__ = lambda self, other: not (self < other)

    return cls


@complete_ordering
class Point:
    def __init__(self, x, y):
        self.x = x
        self.y = y

    def __abs__(self):
        return math.sqrt(self.x**2 + self.y**2)

    def __repr__(self):
        return "Point({0},{1})".format(self.x, self.y)

    def __eq__(self, other):
        if isinstance(other, Point):
            return self.x == other.x and self.y == other.y
        else:
            return NotImplemented

    def __lt__(self, other):
        if isinstance(other, Point):
            return abs(self) < abs(other)
        else:
            return NotImplemented


p1, p2, p3 = Point(2, 3), Point(2, 3), Point(0, 0)
print(p1 <= p2, p1 >= p2, p1 > p3)  # True True True

# argparse

parse command line option.

import argparse

parser = argparse.ArgumentParser()
parser.add_argument("--h", help="This is Hub id", dest="hub_id")
parser.add_argument("--t", help="This is file type", dest="file_type")

args = parser.parse_args()
print(args.hub_id, args.file_type)


# python argparse_test.py --h MAR --t csv
# MAR csv

# rsplit

rsplit 从右向左切割，和 spit 类似，rsplit () 默认分隔符为所有空字符，包括空格、换行 (\n)、制表符 (\t) 等。

s = "this is string example....wow!!!"

print(s.rsplit())
print(s.rsplit('i', maxsplit=2))
print(s.rsplit('w'))
print(s.rsplit(None, 2))

# ['this', 'is', 'string', 'example....wow!!!']
# ['this ', 's str', 'ng example....wow!!!']
# ['this is string example....', 'o', '!!!']
# ['this is', 'string', 'example....wow!!!']

# glob and iglob

glob — Unix style pathname pattern expansion

The glob module finds pathnames using pattern matching rules similar to the Unix shell. No tilde expansion is done, but * , ? , and character ranges expressed with [] will be correctly matched. This is done by using the os.scandir() and fnmatch.fnmatch() functions in concert, and not by actually invoking a subshell.

Note: The pathnames are returned in no particular order. If you need a specific order, sort the results.

Character	Description	Example Pattern	Matches
`*`	Matches zero or more characters in a filename segment.	`*.txt`	`file.txt` , `data1.txt`
`?`	Matches exactly one character in a filename segment.	`data?.csv`	`data1.csv` , `dataA.csv`
`[]`	Matches any single character within the brackets; a hyphen indicates a range.	`file[0-9].jpg`	`file1.jpg` , `file9.jpg`
`**`	Matches any files and zero or more directories when `recursive=True` is set.	`*/.py`	`script.py` , `src/module.py`

iglob

Return an iterator which yields the same values as glob() without actually storing them all simultaneously.

import glob
g = glob.iglob('*.txt')
print(g, list(g)) # <generator object _iglob at 0x00000000025CB4C0> ['combine.txt', 'test.txt']

g = glob.iglob('**/*.txt', recursive=True)
print(list(g)) # ['combine.txt', 'test.txt', 'a\\b\\c.txt']

# keyword argument and keyword-only argumnet

在函数 a 中，x 和 y 是位置参数（positional）同时也可以作为关键字参数（keyword），在函数 b, c 中 y 是仅限关键字参数（keword-only）即强制关键字参数，因为在 y 之前存在 * , 需要指明 y 的值。

def a(x, y):
    pass

a(1, 2)
a(x=1, y=2)

def b(x, *, y):
    pass

# b(1, 2) # TypeError: b() missing 1 required keyword-only argument: 'y'
b(1, y=2)

def c(x, *z, y): # z is Arbitrary Positional Arguments 
    print(x, z, y)

# c(1, 4) # TypeError: c() missing 1 required keyword-only argument: 'y'
c(1, y=4) # 1 () 4
c(1, 2, 3, y=4) # 1 (2, 3) 4

# lambda free variable

lambda 表达式中的 x 是一个自由变量，在运行时绑定值，而不是定义时就绑定，这跟函数的默认值参数定义是不同的。因此，在调用 lambda 表达式的时候，x 的值是执行时的值。如果你想让某个匿名函数在定义时就捕获到值，可以将那个参数值定义成默认参数即可

x = 10
a = lambda y: x + y
x = 20
b = lambda y: x + y
print(a(10)) # 30
print(b(10)) # 30


x = 10
a = lambda y, x=x: x + y
x = 20
b = lambda y, x=x: x + y
print(a(10)) # 20
print(b(10)) # 3


funcs = [lambda x: x + n for n in range(3)]
for func in funcs:
    print(func(1))
# 3
# 3
# 3

funcs = [lambda x, n=n: x + n for n in range(3)]
for func in funcs:
    print(func(1))
# 1
# 2
# 3

# date

import time

now = time.time()
print(now)

tl = time.localtime(now)
print(tl)

date = time.strftime("%Y-%m-%d %H:%M:%S", tl)
print(date)


dnow = "2017-10-16 12:34:00"
timestamp = time.mktime(time.strptime(dnow, "%Y-%m-%d %H:%M:%S"))
print(timestamp)

# 1563515277.4810107
# time.struct_time(tm_year=2019, tm_mon=7, tm_mday=19, tm_hour=13, tm_min=47, tm_sec=57, tm_wday=4, tm_yday=200, tm_isdst=0)
# 2019-07-19 13:47:57
# 1508128440.0

# int

import fractions
import string


print(int(True), int(False))  # 1 0


a = fractions.Fraction(22, 7)
print(a, float(a), int(a))  # 22/7 3.142857142857143 3

print(int("101", 2), int("fF", 16))  # 5 255
print(0b101, 0xFF)  # 5 255
print(bin(5), hex(255))  # 0b101 0xff

print(int("1412", 5))  # 232

# 1412
print(232 // 5, 232 % 5)  # 46 2
print(46 // 5, 46 % 5)  # 9 1
print(9 // 5, 9 % 5)  # 1 4
print(1 // 5, 1 % 5)  # 0 1


def from_base10(n, b):
    if b < 2:
        raise ValueError("Base b must be greater than or equal to 2")

    if n < 0:
        raise ValueError("Number n must be greater than or equal to 0")

    if n == 0:
        return [0]

    digits = []
    while n > 0:
        # m = n % b
        # n = n // b
        # m, n = n % b, n // b
        n, m = divmod(n, b)
        digits.insert(0, m)

    return digits


print(from_base10(232, 5))  # [1, 4, 1, 2]


def encode(digits, digit_map):
    if max(digits) >= len(digit_map):
        raise ValueError("digit_map is not long enough to encode the digits")

    return "".join([digit_map[d] for d in digits])


print(encode([15, 15], "0123456789ABCDEF"))  # FF

print(
    string.ascii_letters, len(string.ascii_letters)
)  # abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ 52


def refrom_base10(number, base):
    digit_map = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
    if base < 2 or base > 62:
        raise ValueError(f"Invalid base {base}, base must be in [2, 62]")

    sign = -1 if number < 0 else 1
    number *= sign

    digits = from_base10(number, base)
    encoding = encode(digits, digit_map)

    if sign == -1:
        encoding = "-" + encoding

    return encoding


print(refrom_base10(232, 5))  # 1412
print(refrom_base10(232, 62))  # 3K
print(refrom_base10(1234567890, 62))  # 1ly7vk

# boolean

There are not many values that evaluate to False, except empty values, such as (), [], {}, “”, the number 0, and the value None. And of course the value False evaluates to False.

The following will return False:

bool(False)
bool(None)
bool(0)
bool(0.0)
bool(Decimal(0))
bool(0 + 0j)
bool("")
bool(())
bool([])
bool({})

One more value, or object in this case, evaluates to False, and that is if you have an object that is made from a class with a __len__ function that returns 0 or False:

class myclass():
  def __len__(self):
    return 0

myobj = myclass()
print(bool(myobj))  # False

if myobj:
    print(True)  # not execute

python xml.etree returns Element object len is 0 if the element’s not has child elements. So don’t use if a: or if not a: to determine if an element exists

import xml.etree.ElementTree

config = xml.etree.ElementTree.parse('.\cfgEDIASNackFileImporter.xml').getroot()

receiver = config.find("./Notification/Appender[@Name='MailTest']/Receiver")

print(len(receiver), boolen(receiver)) # 0 False

<Config>
    <Notification>
        <SMTP Server="xxx.xxx" />

        <Appender Name="MailTest">
            <Receiver value="test@example.com"/>
        </Appender>
    </Notification>
</Config>

boolean is subclass of int

print(issubclass(bool, int)) # True
print(isinstance(True, bool), isinstance(True, int)) # True True

print(True == 1, False == 0)  # True True
print(int(True), int(False))  # 1 0
print(True > False) # True

print(True + True + True, (True + True + True) % 2, -True, 100 * False) # 3 1 -1 0

print(0 or 0, 0 or 1, 1 or 0, 1 or 1)  # 0 1 1 1
print(0 and 0, 0 and 1, 1 and 0, 1 and 1)  # 0 0 0 1

# decimal

import decimal


x = decimal.Decimal("1.25")
y = decimal.Decimal("1.35")

print(decimal.getcontext())  # Context(prec=28, rounding=ROUND_HALF_EVEN, Emin=-999999, Emax=999999, capitals=1, clamp=0, flags=[], traps=[InvalidOperation, DivisionByZero, Overflow])
with decimal.localcontext() as ctx:
    ctx.prec = 6
    ctx.rounding = decimal.ROUND_HALF_UP
    print(decimal.getcontext())  # Context(prec=6, rounding=ROUND_HALF_UP, Emin=-999999, Emax=999999, capitals=1, clamp=0, flags=[], traps=[InvalidOperation, DivisionByZero, Overflow])

    print(round(x, 1)) # 1.3
    print(round(y, 1)) # 1.4

print(round(x, 1)) # 1.2
print(round(y, 1)) # 1.4

# ROUND_HALF_EVEN: Round to nearest with ties going to nearest even integer
print(round(1.25, 1), round(1.35, 1)) # 1.2 1.4
print(round(1.5), round(2.5)) # 2 2

import sys
from decimal import Decimal

a = 3.1415
b = Decimal("3.1415")
print(sys.getsizeof(a), sys.getsizeof(b)) # 24 120

from decimal import Decimal
import math
import time


def run_float(n=1):
    a = 3.1415
    for _ in range(n):
        math.sqrt(a)


def run_decimal(n=1):
    a = Decimal("3.1415")
    for _ in range(n):
        a.sqrt()


start = time.perf_counter()
run_float(10000000)
end = time.perf_counter()
print("float", end - start) # float 0.30956040002638474

start = time.perf_counter()
run_decimal(10000000)
end = time.perf_counter()
print("decimal", end - start) # print("decimal", end - start)

# float

import math


a = 0.3
b = 0.1 + 0.1 + 0.1

print(a == b, math.isclose(a, b))  # False True
print(format(a, ".25f"), format(b, ".25f"))  # 0.2999999999999999888977698 0.3000000000000000444089210


x = 12345678.01
y = 12345678.02
print(math.isclose(x, y, rel_tol=0.01)) # True


x = 0.01
y = 0.02
print(math.isclose(x, y, rel_tol=0.01)) # False

x = 0.00000001
y = 0.00000002
print(math.isclose(x, y, rel_tol=0.01), math.isclose(x, y, rel_tol=0.01, abs_tol=0.01)) # False True


x = 0.00000001
y = 0.00000002

a = 12345678.01
b = 12345678.02

print(
    math.isclose(x, y, rel_tol=0.01, abs_tol=0.0001),
    math.isclose(a, b, rel_tol=0.01, abs_tol=0.0001),
) # True True

1
2
3

# ROUND_HALF_EVEN: Round to nearest with ties going to nearest even integer
print(round(1.5), round(2.5))  # 2 2
print(round(-1.5), round(-2.5))  # -2 2

# Fraction

from fractions import Fraction
import math


print(Fraction(1), repr(Fraction(1)))  # 1, Fraction(1, 1)
print(Fraction(1, 2), Fraction(denominator=2, numerator=1))  # 1/2 1/2


x = Fraction(math.pi)
print(x, x.limit_denominator(10))  # 884279719003555/281474976710656 22/7

x = Fraction(0.3)
print(x, x.limit_denominator(10))  # 5404319552844595/18014398509481984 3/10

# timeit

import timeit

TIMES = 10000

SETUP = """
symbols = '$¢£¥€¤'
def non_ascii(c):
    return c > 127
"""

# Signature: repeat(stmt='pass', setup='pass', timer=<built-in function perf_counter>, repeat=3, number=1000000, globals=None)
def clock(label, cmd):
    res = timeit.repeat(cmd, setup=SETUP, number=TIMES)
    print(label, *('{:.3f}'.format(x) for x in res))

clock('listcomp        :', '[ord(s) for s in symbols if ord(s) > 127]')
clock('listcomp + func :', '[ord(s) for s in symbols if non_ascii(ord(s))]')
clock('filter + lambda :', 'list(filter(lambda c: c > 127, map(ord, symbols)))')
clock('filter + func   :', 'list(filter(non_ascii, map(ord, symbols)))')

# listcomp        : 0.003 0.002 0.002 0.002 0.003
# listcomp + func : 0.004 0.004 0.003 0.004 0.004
# filter + lambda : 0.005 0.005 0.005 0.006 0.005
# filter + func   : 0.004 0.004 0.005 0.004 0.004

# eval

功能：将字符串 str 当成有效的表达式来求值并返回计算结果。

语法： eval (source [, globals [, locals]]) -> value

参数：

source：一个 python 表达式或函数 compile () 返回的代码对象。
globals：可选。必须是 dictionary。
ocals：可选。任意 map 对象。

可以把 list, tuple, dict 和 string 相互转化。

a = "[[1,2], [3,4], [5,6], [7,8], [9,0]]"
print(type(a))  # <class 'str'>

b = eval(a)
print(type(b))  # <class 'list'>

# uuid

uuid: 通用唯一标识符（Universally Unique ID，UUID）

import uuid
user_id = uuid.uuid4()
print(user_id)
# cb79af9a-b05f-49a7-acd2-2675852e12f9

from datetime import datetime

def next_id():
    return '%015d%s000' % (int(time.time() * 1000), uuid.uuid4().hex)
print(next_id())  # 001583823607352451339f583754d75b479e67714a227ea000
print('%015d' % 10) # '000000000000010'

# partial

from functools import partial

# 默认按十进制转换
r1 = int("12")
print(r1, type(r1))
# 12 <class 'int'>

# 按二进制转换
r2 = int("0101", base=2)
print(r2, type(r2))
# 5 <class 'int'>

# 使用偏函数, 改造原有的 int 函数
int2 = partial(int, base=2)
r3 = int2("0101")
print(r3, type(r3))
# 5 <class 'int'>


def multiply(a: float, b: float, name: str | None) -> float:
    if name is not None:
        print(f"{name=} ({a=}, {b=})")

    return a * b

double = partial(multiply, 2, name="Double")
triple = partial(multiply, b=3, name="Triple")

print(double(10))
print(triple(10))

# name='Double' (a=2, b=10)
# 20
# name='Triple' (a=10, b=3)
# 30

class Partial:
    def __init__(self, func, *args):
        self._func = func
        self._args = args

    def __call__(self, *args):
        return self._func(*self._args, *args)


def fn(a, b, c):
    return a, b, c


partial_fn = Partial(fn, 10, 20)
print(partial_fn(30))  # (10, 20, 30)

# base64

import base64
s = 'hello'
# b64encode: Encode the bytes-like object s using Base64 and return a bytes object.
a = base64.b64encode(s.encode())
print(a) # b'aGVsbG8='

# encode: Encode a file; input and output are binary files.
with open('01.png', 'rb') as r, open('01.txt', 'wb') as w:
    base64.encode(r, w)

# sqrt

1
2
3

import math

print(math.sqrt(25) == 25 ** 0.5) # True

# mod

假设 q 是 a、b 相除产生的商 (quotient)，r 是相应的余数 (remainder)，那么在几乎所有的计算系统中，都满足：a = b x q + r，其中 |r|<|a|。所以 r = a - (a / b) x b

python 使用 floor 向下整除法

print(-17 // 10, 17 // -10, -17 // -10) # # -2 -2 1

# -17 % 10 = -17 - (-17 // 10) * 10 = -17 - (-2) * 10 = 3
print(-17 % 10, 17 % -10, -17 % -10) # 3 -3 -7

JavaScript 使用向上整除法

1 2	// -17 % 10 = -17 - parseInt(-17 / 10) * (-10) = -17 - (-1) * (-10) = -7 console.log(-17 % 10, 17 % -10, -17 % -10); // -7 7 -7

# max count

nums = [3, 3, 5, 6, 5, 6, 3, 2, 1, 3]
most = max(set(nums), key=nums.count)
print(most)  # 3
print(nums.count(3)) # 4

# reduce

from functools import reduce


lst = [5, 8, 6, 10, 9]

min = reduce(lambda x, y: x if x < y else y, lst)
print(min) # 5

sum = reduce(operator.add, lst)
print(sum) # 38

join = reduce(lambda a, b: f"{a}, {b}", lst)
print(join) # 5, 8, 6, 10, 9

lst = [0, "", None, 100]

any_true = reduce(lambda a, b: bool(a) or bool(b), lst)
print(any_true) # True

all_true = reduce(lambda a, b: bool(a) and bool(b), lst)
print(all_true) # False

from operator import mul


def _reduce(fn, sequence):
    result = sequence[0]
    for x in sequence[1:]:
        result = fn(result, x)
    return result


lst = [1, 2, 3, 4]

product = _reduce(lambda x, y: x * y, lst)
print(product) # 24

product = _reduce(mul, lst)
print(product) # 24

# empty function body

def fun1():
    pass

def fun2():
    ...

# Avoid forget to implement
def fun3():
    raise NotImplementedError("Not Implement func3")


fun1()
fun2()
fun3()

# * and / argument

* 后面的参数必须使用关键字参数 (Keyword arguments)。

/ 前面的参数必须使用位置参数 (Positional argulemts)。

def test(a, *, b, c):
    print(a, b, c)
    
test(1, b=2, c=3)
# 1 2 3


def work(name: str, /, job: str, *, hours: int):
    print(f"{name} is {job} for {hours} hours.")

work("Babb", job="coding", hours=10) # Babb is coding for 10 hours.
work("Babb", "coding", hours=10) # Babb is coding for 10 hours.

# argument default value

函数参数的默认值是在创建 / 编译时确定不是调用时。如下两次函数调用输出的时间不变。

from datetime import datetime
import time


def log(msg, current_dt=datetime.now()):
    print(msg, current_dt)


time.sleep(2)
print(datetime.now()) # 2025-04-02 17:43:36.193251

log("error")  # error 2025-04-02 17:43:34.192997
time.sleep(2)
log("warning")  # warning 2025-04-02 17:43:34.192997

# * pack unpack

print(2 * 3)
print("ha" * 3)
print([1, 2] * 3)

numbers = [1, 2, 3, 4, 5]
first, *rest = numbers
print(first, rest)


def print_args(*args):
    print(type(args), args)


print_args(1, 2, 3, 4)


def print_kwargs(**kwargs):
    print(type(kwargs), kwargs)


print_kwargs(a=1, b=2, c=3)


def greet(name, age):
    print(f"hello {name}, you are {age} old.")


person = ("Babb", 30)
greet(*person)


list1 = [1, 2, 3]
tuple1 = (4, 5, 6)
merged = [*list1, *tuple1]
print(merged)


def create_profile(name, age, email):
    print(f"{name} - {age} - {email}")


option = {"name": "Babb", "age": 33, "email": "babb@example.com"}
create_profile(**option)


dict1 = {"a": 1, "b": 2}
dict2 = {"c": 3, "d": 4}
merged = {**dict1, **dict2}
print(merged)

# with context

# f = open("test.txt")
# try:
#     pass
# finally:
#     f.close()

# with open("test.txt") as f:
#     pass


from contextlib import contextmanager
import os
import time


class MyOpen:
    def __init__(self, filepath):
        print("__init__")
        self.filepath = filepath

    def __enter__(self):
        print("__enter__")
        return self.filepath

    def __exit__(self, exc_type, exc_value, traceback):
        print("__exit__")
        if exc_type is ValueError:
            print("Caught a ValueError")
        return True


with MyOpen("test.txt") as f:
    print(f)
    raise ValueError("A ValueError occured")


@contextmanager
def my_open(filepath):
    print("enter")

    try:
        yield filepath
    finally:
        print("exit")


with my_open("test.txt") as f:
    print(f)


@contextmanager
def timer():
    start = time.time()

    yield

    print(f"cost {time.time() - start}")


with timer():
    time.sleep(2)


@contextmanager
def set_temporary_env(**kwargs):
    for k, v in kwargs.items():
        os.environ[k] = v
    yield
    for k, v in kwargs.items():
        del os.environ[k]


print(os.environ.get("TEST_VAR"))

with set_temporary_env(TEST_VAR="xyz"):
    print(os.environ.get("TEST_VAR"))

print(os.environ.get("TEST_VAR"))

# with conext seed random

import contextlib
import random
from typing import Any, Generator

@contextlib.contextmanager
def rand_state(seed: int | None = None) -> Generator:
    state: tuple[Any, ...] = random.getstate()
    random.seed(seed)
    try:
        yield
    finally:
        random.setstate(state)


for _ in range(3):
    with rand_state(1):
        print(random.randrange(0, 100), random.randint(0, 100), random.random())

    print(random.randrange(0, 100), random.randint(0, 100), random.random())
    
# 17 72 0.8474337369372327
# 90 88 0.18780016419984147
# 17 72 0.8474337369372327
# 0 99 0.6800110455387003
# 17 72 0.8474337369372327
# 27 35 0.586171697319297

# walrus operator

(a := 1)
print(a)

lst = [1, 2, 3]
# length = len(lst)
# if length > 0:
#     print(f"list is not empty, its size is {length}")
if (length := len(lst)) > 0:
    print(f"list is not empty, its size is {length}")
print(length)


# while True:
#     cmd = input()
#     if cmd == "exit":
#         break

#     print(f"Got input {cmd}")

while (cmd := input()) != "exit":
    print(f"Got input {cmd}")


lst = [2, 4, 6, 7, 10]
# is_even = True
# for i in lst:
#     if i % 2 != 0:
#         is_even = False
#         break
# print(i)
# print(is_even)

is_even = all((n := i) % 2 == 0 for i in lst)
print(n)
print(is_even)

def get_info(text: str) -> dict:
    return {
        "words": (words := text.split()),
        "word count": len(words),
        "character count": len("".join(words))
    }

print(get_info("Hello World")) # {'words': ['Hello', 'World'], 'word count': 2, 'character count': 10}

# match case

from dataclasses import dataclass


def match_point(point: tuple):
    match point:
        case (0, 0):
            print("Origin!")
        # case (x, 0):
        #     print("on x axis")
        # case (0, y):
        #     print("on y axis")
        case (x, 0) | (0, x):
            print("on axis")
        case (x, y):
            print(f"{x=},{y=}")
        case others:
            print(f"{others} is not valid point")


p1 = (0, 0)
p2 = (1, 2)
p3 = (1, 2, 3)
p4 = (1, 0)
match_point(p1)
match_point(p2)
match_point(p3)
match_point(p4)


my_tuple = (0, 0)
my_list = [0, 0]
match my_tuple:
    case [0, 0]:
        print("matched")

match my_tuple:
    case 0, 0:
        print("matched")

match my_list:
    case (0, 0):
        print("matched")


match my_list:
    case tuple([0, 0]):
        print("matched")
    case _:
        print("not matched")

match my_tuple:
    case tuple:
        print("matched")


def match_quadrant(point):
    match point:
        case (x, y) if x > 0 and y > 0:
            print("First quadrant")
        case (x, y) if x < 0 and y > 0:
            print("Second quadrant")
        case (x, y) if x < 0 and y < 0:
            print("Third quadrant")
        case (x, y) if x > 0 and y < 0:
            print("Fourth quadrant")
        case (x, y):
            print("On axis")


match_quadrant((1, 2))
match_quadrant((-1, 2))
match_quadrant((-1, -2))
match_quadrant((1, -2))


dict_p = {"x": 0, "y": 30}

match dict_p:
    case {"x": 0, "y": 0}:
        print("Origin")
    case {"x": x, "y": y}:
        print(f"{x=}, {y=}")

match dict_p:
    case {"x": 0}:
        print("matched")


class Point:
    __match_args__ = ("x", "y")

    def __init__(self, x, y):
        self.x = x
        self.y = y


p = Point(0, 1)

match p:
    case Point(x=0, y=0):
        print("Origin")
    case Point(x=x, y=y):
        print(f"{x=},{y=}")

match p:
    case Point(0, 0):
        print("Origin")
    case Point(x, y):
        print(f"{x=},{y=}")


@dataclass
class DataPoint:
    x: int
    y: int


p = DataPoint(0, 10)

match p:
    case Point(0, 0):
        print("Origin")
    case Point(x, y):
        print(f"{x=},{y=}")

symbols = {
    "F": "\u2192", 
    "B": "\u2190", 
    "L": "\u2191", 
    "R": "\u2193", 
    "pick": "\u2923", 
    "drop": "\u2925"
}

print(symbols) # {'F': '→', 'B': '←', 'L': '↑', 'R': '↓', 'pick': '⤣', 'drop': '⤥'}

def op(command):
    match command:
        case "move F":
            return symbols["F"]
        case "move B":
            return symbols["B"]
        case "move L":
            return symbols["L"]
        case "move R":
            return symbols["R"]
        case "pick":
            return symbols["pick"]
        case "drop":
            return symbols["drop"]
        case _:
            raise ValueError(f"{command} does not compute!")
            
print([
    op("move F"),
    op("move F"),
    op("move L"),
    op("pick"),
    op("move R"),
    op("move L"),
    op("move F"),
    op("drop"),
]) # ['→', '→', '↑', '⤣', '↓', '↑', '→', '⤥']


def op(command):
    match command:
        case ["move", ("F" | "B" | "L" | "R") as direction]:
            return symbols[direction]
        case "pick":
            return symbols["pick"]
        case "drop":
            return symbols["drop"]
        case _:
            raise ValueError(f"{command} does not compute!")

print([
    op(["move", "F"]),
    op(["move", "F"]),
    op(["move", "L"]),
    op("pick"),
    op(["move", "R"]),
    op(["move", "L"]),
    op(["move", "F"]),
    op("drop"),
]) # ['→', '→', '↑', '⤣', '↓', '↑', '→', '⤥']


def op(command):
    match command:
        case ["move", *directions]:
            return tuple(symbols[direction] for direction in directions)
        case "pick":
            return symbols["pick"]
        case "drop":
            return symbols["drop"]
        case _:
            raise ValueError(f"{command} does not compute!")
        
print([
    op(["move", "F", "F", "L"]),
    op("pick"),
    op(["move", "R", "L", "F"]),
    op("drop"),
]) # [('→', '→', '↑'), '⤣', ('↓', '↑', '→'), '⤥']


def op(command):
    match command:
        case ["move", *directions] if set(directions) < symbols.keys():
            return tuple(symbols[direction] for direction in directions)
        case "pick":
            return symbols["pick"]
        case "drop":
            return symbols["drop"]
        case _:
            raise ValueError(f"{command} does not compute!")

print(op(["move", "F", "F", "L"])) # ('→', '→', '↑')
try:
    op(["move", "up"])
except Exception as ex:
    print(type(ex), ex) # <class 'ValueError'> ['move', 'up'] does not compute!

# magic method

from __future__ import annotations
from typing import List


class ShoppingCart:
    def __init__(self, items: List[str]):
        self.items = items

    def __add__(self, another_cart: ShoppingCart) -> ShoppingCart:
        return ShoppingCart(self.items + another_cart.items)

    def __str__(self) -> str:
        return f"Cart({self.items})"

    def __len__(self) -> int:
        return len(self.items)

    def __call__(self, item: str):
        self.items.append(item)


cart = ShoppingCart(["apple", "orange"])
cart("banana")
print(cart)
print(len(cart))

cart = cart + ShoppingCart(["grape", "peach"])
print(cart)


class add:
    def __init__(self, num):
        self.num = num

    def __str__(self):
        return str(self.num)

    def __add__(self, other):
        return self.num + other

    def __call__(self, other):
        # print(id(self))
        return add(self.num + other)

        # NOT
        # self.num += other
        # return self


addTwo = add(2)
print(addTwo)
print(addTwo + 5)

print(addTwo(3))
print(addTwo(3)(5))


print(add(1)(2)(3)(4)(5))

# unitest

bank_account.py

import requests


class BankAccount:
    def __init__(self, initial_balance=0):
        if initial_balance < 0:
            raise ValueError("Initial balance cannot be negative")
        self.balance = initial_balance

    def deposit(self, amount):
        if amount < 0:
            raise ValueError("Deposit amount cannot be negative")
        self.balance += amount

    def withdraw(self, amount):
        if amount < 0:
            raise ValueError("Withdraw amount cannot be negative")
        self.balance -= amount

    def get_interest_rate(self):
        url = ""
        response = requests.get(url)
        if response.status_code == 200:
            return response.json().get("rate")
        else:
            raise Exception("Failed to fetch interest rate")

test_bank_account.py

import unittest
from unittest.mock import patch
from bank_account import BankAccount


class TestBankAccount(unittest.TestCase):
    @classmethod
    def setUpClass(cls):
        print("setUpClass")

    @classmethod
    def tearDownClass(cls):
        print("tearDownClass")

    def setUp(self):
        self.account = BankAccount(100)

    def tearDown(self):
        print("tearDown")

    def test_create_account(self):
        self.account = BankAccount(100)
        self.assertIsInstance(self.account, BankAccount)

        with self.assertRaises(ValueError):
            BankAccount(-100)

    def test_deposit(self):
        self.account = BankAccount(100)
        self.account.deposit(50)
        self.assertEqual(
            self.account.balance, 150, msg="Balance should be 100 + 50 after deposit"
        )

        with self.assertRaises(
            ValueError, msg="Negative deposit should raise ValueError"
        ):
            self.account.deposit(-50)

    def test_withdraw(self):
        self.account = BankAccount(100)
        self.account.withdraw(50)
        self.assertEqual(
            self.account.balance, 50, msg="Balance should be 100 - 50 after withdraw"
        )

        with self.assertRaises(
            ValueError, msg="Negative withdraw should raise ValueError"
        ):
            self.account.deposit(-50)

    def test_get_interest_rate(self):
        with patch("bank_account.requests.get") as mock_get:
            mock_get.return_value.status_code = 200
            mock_get.return_value.json.return_value = {"rate": 0.5}
            interest_rate = self.account.get_interest_rate()
            self.assertEqual(interest_rate, 0.5)


if __name__ == "__main__":
    unittest.main()

unittest mock

from unittest.mock import Mock, MagicMock, patch

mock = Mock()
mock.get.return_value = "mocked value"
print(mock.get())  # Output: mocked value


magic_mock = MagicMock()
# magic_mock.__str__.return_value = "magic mocked value"
magic_mock.__str__ = lambda self=magic_mock: "magic mocked value"
print(str(magic_mock))  # Output: magic mocked value


def foo():
    return "original value"


with patch("__main__.foo", return_value="patched value"):
    print(foo())  # Output: patched value

with patch("__main__.foo") as mock_foo:
    mock_foo.return_value = "another patched value"
    print(foo())  # Output: another patched value


@patch("__main__.foo")
def test_foo(mock_foo):
    mock_foo.return_value = "test patched value"
    print(foo())  # Output: test patched value


test_foo()

# reference counting

import sys
import ctypes


a = [1, 2, 3, 4] # +1
b = a # +1
print(sys.getrefcount(a)) # +1, 3


def ref_count(address: int):
    return ctypes.c_long.from_address(address).value


print(ref_count(id(a))) # not + 1, 2

b = None # -1

print(ref_count(id(a))) # 1

a = None # -1

print(ref_count(id(a))) # -1?

# pre-caculate at compile time

def func1():
    a = 24 * 60
    b = (1, 2) * 5
    c = "abc" * 3
    d = "abc" * 10000
    e = "the quick brown fox " * 5
    f = ["a", "b"] * 3

print(func1.__code__.co_consts)
# (None, 1440, (1, 2, 1, 2, 1, 2, 1, 2, 1, 2), 'abcabcabc', 'abc', 10000, 'the quick brown fox the quick brown fox the quick brown fox the quick brown fox the quick brown fox ', 'a', 'b', 3)

def func2(e):
    for e in [1, 2, 3]:
        pass

print(func2.__code__.co_consts)  # (None, (1, 2, 3))


def func3(e):
    for e in {1, 2, 3}:
        pass

print(func3.__code__.co_consts)  # (None, frozenset({1, 2, 3}))

# not logic in `main`

# helps to avoid cluttering the global namespace
def main() -> None:
    local_a = "a"
    print(f"{globals().get("global_a")} {globals().get("local_a")}") # A None

if __name__ == "__main__":
    global_a = "A"
    main()

# generator type

A generator can be annotated using the generic type Generator[YieldType, SendType, ReturnType]

import sys
from typing import Generator


def fibonacci_generator() -> Generator[int, None, None]:
    a, b = 0, 1
    while True:
        yield a
        a, b = b, a + b

fib_gen: Generator[int, None, None] = fibonacci_generator()
n: int = 10
line_break: str = "-" * 20

while True:
    input(f"Tap 'enter' for the next {n} number of fibonacci")
    print(line_break)
    for _ in range(n):
        print(next(fib_gen))
    print(line_break)

def read(path: str) -> Generator[str, None, str]:
    with open(path, "r") as file:
        for line in file:
            yield line.strip()

    return "This is end!"

reader: Generator[str, None, str] = read("note.txt")

input("Press 'enter'")

while True:
    try:
        print(next(reader))
    except StopIteration as e:
        print(e.value)
        sys.exit()
    input()

def cumulative_sum() -> Generator[float, float, None]:
    total: float = 0
    while True:
        total += yield total


cumulative_generator: Generator[float, float, None] = cumulative_sum()
next(cumulative_generator)

while True:
    value: float = float(input("Enter a value: "))
    current_sum: float = cumulative_generator.send(value)
    print(f"{current_sum=}")

def infinite_repeater(sequence: list[Any]) -> Generator[Any, None, None]:
    while True:
        for item in sequence:
            yield item


repeater: Generator[Any, None, None] = infinite_repeater([1, 2, 3, 4])

for _ in range(10):
    print(next(repeater))

# methodcaller

from operator import methodcaller
from timeit import repeat

names: list[str] = ["Bob", "Tom", "James", "Sandra", "Blake"]
start_with_b: methodcaller = methodcaller("startswith", "B")

filtered: filter = filter(start_with_b, names)
print(list(filtered)) # ['Bob', 'Blake']

count_a: methodcaller = methodcaller("count", "a")
print(list(sorted(names, key=count_a))) # ['Bob', 'Tom', 'James', 'Blake', 'Sandra']


warm_up: str = """
for i in range(5):
    ...
"""

methodcaller_test: str = """
filter(start_with_b, names)
"""

lambda_test = """
filter(lambda x: x.startswith("B"), names)
"""

repeat(stmt=warm_up)

methodcaller_time: float = min(repeat(methodcaller_test, globals=globals()))
lambda_time: float = min(repeat(lambda_test, globals=globals()))

print(f"{methodcaller_time=:.5f}") # methodcaller_time=0.04031
print(f"{lambda_time=:.5f}") # lambda_time=0.07081

# lru_cache

from functools import lru_cache

@lru_cache(maxsize=128, typed=False)
def add(a, b):
    print(f"{a} + {b} = {a + b}")
    return a + b

add(1, 1)
add(1, 1) # use cache
add(1.0, 1.0) # typed=False use cache 
add(1, 2) 
add(2, 1) # order change not use cache
print(add.cache_info())

# 1 + 1 = 2
# 1 + 2 = 3
# 2 + 1 = 3
# CacheInfo(hits=2, misses=3, maxsize=128, currsize=3)



@lru_cache(maxsize=2, typed=True)
def add(a, b):
    print(f"{a} + {b} = {a + b}")
    return a + b

add(1, 1) # 1
add(1.0, 1.0) # 2 typed=True not use cache
add(1, 2) # 3 > 2 maxsize, add(1, 2) cached, add(1, 1) not cached
add(1, 2)
add(1, 1) # not use cache
print(add.cache_info())
print(add.cache_clear())
print(add.cache_info())

# 1 + 1 = 2
# 1.0 + 1.0 = 2.0
# 1 + 2 = 3
# 1 + 1 = 2
# CacheInfo(hits=1, misses=4, maxsize=2, currsize=2)
# None
# CacheInfo(hits=0, misses=0, maxsize=2, currsize=0)

fibonacci

from functools import lru_cache

@lru_cache
def fibonacci(n): 
    print ('Calculating fibonacci({0})'.format(n))
    return 1 if n < 3 else fibonacci(n-1) + fibonacci(n-2)

print(fibonacci(6))
print(fibonacci(6))
print(fibonacci(8))

# Calculating fib(6)
# Calculating fib(5)
# Calculating fib(4)
# Calculating fib(3)
# Calculating fib(2)
# Calculating fib(1)
# 8
# 8
# Calculating fib(8)
# Calculating fib(7)
# 21

simple custom define memolizer like lru_cache

def memolizer(fn):
    cache = dict()
    
    def inner(n):
        if n not in cache:
            cache[n] = fn(n)

        return cache[n]
    
    return inner

@memolizer
def fib(n): 
    print ('Calculating fib({0})'.format(n))
    return 1 if n < 3 else fib(n-1) + fib(n-2)

print(fib(6))
print(fib(6))

# Calculating fib(6)
# Calculating fib(5)
# Calculating fib(4)
# Calculating fib(3)
# Calculating fib(2)
# Calculating fib(1)
# 8
# 8

simple custom define memolizer using frozenset support order insensitive cache

from functools import lru_cache

@lru_cache
def add(a, b):
    print("caculating a + b")
    return a + b


print(add(a=1, b=2))
print(add(b=2, a=1)) # The order not equal, not use cache
# caculating a + b
# 3
# caculating a + b
# 3

# arguments must be hashable
print(add(a=[1, 2, 3], b=[4, 5, 6])) # TypeError: unhashable type: 'list'

def memolizer(fn):

    cache = {}

    def inner(*args, **kwargs):
        key = frozenset(args) | frozenset(kwargs.items()) # frozenset is no order
        if key not in cache:
            cache[key] = fn(*args, **kwargs)
        return cache[key]

    return inner

@memolizer
def add(a, b):
    print("caculating a + b")
    return a + b

print(add(a=1, b=2))
print(add(b=2, a=1)) 
# caculating a + b
# 3
# 3

print(add(1, 2))
print(add(2, 1))
# caculating a + b
# 3
# 3

cache dict as default parameter

def factorial(n, cache={}):
    if n < 1:
        return 1
    elif n in cache:
        return cache[n]
    else:
        print(f"caculating {n}")
        result = n * factorial(n - 1, cache=cache)
        cache[n] = result
        return result


print(factorial(n=3))
# caculating 3
# caculating 2
# caculating 1
# 6

print(factorial(n=4))
# caculating 4
# 24

# cached_property

from functools import cached_property
from statistics import mean


class Caculator:
    def __init__(self, values: list[float]):
        self.values = values

    @cached_property
    def sum(self) -> float:
        print(f"Getting the sum of: {self.values}")
        return sum(self.values)
    
    @cached_property
    def average(self) -> float:
        print(f"Getting the average of: {self.values}")
        return mean(self.values)
    
numbers: list[float] = [1.5, 2.5, 3.5, 4.5, 5.5]
caculator: Caculator = Caculator(numbers)
# print(caculator.average()) # TypeError: 'float' object is not callable
print(caculator.average)
print(caculator.average)
print(caculator.average)
# Getting the average of: [1.5, 2.5, 3.5, 4.5, 5.5]
# 3.5
# 3.5
# 3.5

caculator: Caculator = Caculator(numbers + [6.5, 7.5])
print(caculator.average)
print(caculator.average)
print(caculator.average)
# Getting the average of: [1.5, 2.5, 3.5, 4.5, 5.5, 6.5, 7.5]
# 4.5
# 4.5
# 4.5

# singledispatch

from html import escape
# from functools import singledispatch, singledispatchmethod

def singledispatch(fn):
    registry = dict()

    registry[object] = fn
    
    def register(type_):
        def inner(fn):
            registry[type_] = fn
            return fn  # we do this so we can stack register decorators!
        return inner
   
    def decorator(arg):
        fn = registry.get(type(arg), registry[object])
        return fn(arg)
    
    def dispatch(type_):
        return registry.get(type_, registry[object])

    decorator.register = register
    decorator.registry = registry.keys()
    decorator.dispatch = dispatch
    return decorator


@singledispatch
def htmlize(arg):
    return escape(str(arg))

@htmlize.register(float)
def html_real(a):
    return '{0:.2f}'.format(round(a, 2))

@htmlize.register(str)
def html_str(s):
    return escape(s).replace('\n', '<br/>\n')

@htmlize.register(tuple)
@htmlize.register(list)
def html_list(l):
    items = ['<li>{0}</li>'.format(htmlize(item)) for item in l]
    return '<ul>\n' + '\n'.join(items) + '\n</ul>'

@htmlize.register(dict)
def html_dict(d):
    items = ['<li>{0}={1}</li>'.format(htmlize(k), htmlize(v)) for k, v in d.items()]
    return '<ul>\n' + '\n'.join(items) + '\n</ul>'

print(htmlize.registry) # dict_keys([<class 'object'>, <class 'float'>, <class 'str'>, <class 'list'>, <class 'tuple'>, <class 'dict'>])

print(htmlize([1, 2, 3]))
# <ul>
# <li>1</li>
# <li>2</li>
# <li>3</li>
# </ul>

print(htmlize("""this
is a multi line string with
a < 10"""))
# this<br/>
# is a multi line string with<br/>
# a &lt; 10

# callable

print(
    callable(print),
    callable("abc".upper),
    callable(str.upper),
    callable(callable),
    callable(lambda x: x.upper()),
) # True True True True True


class MyClass:
    def __init__(self, x=0):
        self.counter = x

    def __call__(self, x=1):
        print("update counter")
        self.counter = x


a = MyClass()  # MyClass(), MyClass is callable
print(callable(MyClass)) # True
print(callable(a)) # True

# operator

import operator

print(dir(operator))


print(
    operator.add(2, 3),
    operator.mul(2, 3),
    operator.truediv(3, 2),
    operator.floordiv(3, 2),
)  # 5 6 1.5 1

print(operator.lt(2, 3), operator.le(2, 2))  # True True

a = 1
b = a
print(operator.is_(a, b))  # True

print(operator.truth([]), operator.truth("abc"))  # False True


lst = [1, 2, 3, 4]
print(operator.getitem(lst, 1))  # 2
operator.setitem(lst, 1, 100)
operator.delitem(lst, 3)
print(lst)  # [1, 100, 3]


lst = [1, 2, 3, 4]
fun = operator.itemgetter(1)
print(fun)  # operator.itemgetter(1)
print(fun(lst), fun("python"))  # 2 y


lst = [1, 2, 3, 4]
fun = operator.itemgetter(1, 2, 3)
print(fun(lst), fun("python"))  # (2, 3, 4) ('y', 't', 'h')


class MyClass:
    def __init__(self):
        self.a = 1
        self.b = 2
        self.c = 3

    def test(self):
        print("test method running")


obj = MyClass()

prop_a = operator.attrgetter("a")
print(prop_a(obj))  # 1

var = "b"
print(operator.attrgetter(var)(obj))  # 2

a, b, test = operator.attrgetter("a", "b", "test")(obj)
print(a, b)  # 1 2
test()  # test method running


lst = [5 - 10j, 3 + 3j, 2 - 100j]
sorted_result = sorted(lst, key=operator.attrgetter("real"))
print(sorted_result)  # [(2-100j), (3+3j), (5-10j)]


lst = [(2, 3, 4), (1, 3, 5), (6,), (4, 100)]
sorted_result = sorted(lst, key=lambda x: x[0])
print(sorted_result)  # [(1, 3, 5), (2, 3, 4), (4, 100), (6,)]
sorted_result = sorted(lst, key=operator.itemgetter(0))
print(sorted_result)  # [(1, 3, 5), (2, 3, 4), (4, 100), (6,)]


class MyClass:
    def __init__(self):
        self.a = 1
        self.b = 2

    def test(self, c, d, *, e):
        print(self.a, self.b, c, d, e)


obj = MyClass()

fun = operator.attrgetter("test")
fun(obj)(3, 4, e=5)  # 1 2 3 4 5


operator.methodcaller("test", 30, 40, e=50)(obj)  # 1 2 30 40 50

# validate arguments provided

def validate(a=object(), b=object(), *, kw=object()):
    default_a = validate.__defaults__[0]
    default_b = validate.__defaults__[1]
    default_kw = validate.__kwdefaults__["kw"]

    if a is not default_a:
        print("Argument a was provided")
    else:
        print("Argument a was not provided")

    if b is not default_b:
        print("Argument b was provided")
    else:
        print("Argument b was not provided")

    if kw is not default_kw:
        print("Argument kw was provided")
    else:
        print("Argument kw was not provided")


validate(100, 200, kw=None)
# Argument a was provided
# Argument b was provided
# Argument kw was provided

validate(100, 200)
# Argument a was provided
# Argument b was provided
# Argument kw was not provided

validate()
# Argument a was not provided
# Argument b was not provided
# Argument kw was not provided

# reference counting

import sys
import ctypes


a = [1, 2, 3, 4]
b = a
print(sys.getrefcount(a))  # 3 sys.getrefcount also reference a


def ref_count(address: int):
    return ctypes.c_long.from_address(address).value


print(ref_count(id(a)))  # 2

b = None
print(ref_count(id(a)))  # 1

a = None
print(ref_count(id(a)))  # -1

# pascal triangle

from math import factorial
from pprint import pprint


def combination(n, k):
    return factorial(n) // (factorial(k) * factorial(n - k))


size = 10
pascal = [[combination(n, k) for k in range(n + 1)] for n in range(size + 1)]
pprint(pascal)

# [[1],
#  [1, 1],
#  [1, 2, 1],
#  [1, 3, 3, 1],
#  [1, 4, 6, 4, 1],
#  [1, 5, 10, 10, 5, 1],
#  [1, 6, 15, 20, 15, 6, 1],
#  [1, 7, 21, 35, 35, 21, 7, 1],
#  [1, 8, 28, 56, 70, 56, 28, 8, 1],
#  [1, 9, 36, 84, 126, 126, 84, 36, 9, 1],
#  [1, 10, 45, 120, 210, 252, 210, 120, 45, 10, 1]]

# Loop

# while loop

# min_length = 2
# name = input("Please enter your name: ")

# while not (len(name) > min_length and name.isprintable() and name.isalpha()):
#     name = input("Please enter your name: ")

# print("Hello, {0}".format(name))


min_length = 2

while True:
    name = input("Please enter your name: ")

    if len(name) > min_length and name.isprintable() and name.isalpha():
        break

print("Hello, {0}".format(name))


# lst = [1, 2, 3]
# val = 10
# found = False
# idx = 0
# while idx < len(lst):
#     if lst[idx] == val:
#         found = True
#         break
#     idx += 1

# if not found:
#     lst.append(val)

# print(lst)


lst = [1, 2, 3]
val = 10
idx = 0
while idx < len(lst):
    if lst[idx] == val:
        break
    idx += 1
else:
    lst.append(val)

print(lst) # [1, 2, 3, 10]

# for loop

s = "hello"
i = 0
for c in s:
    print(i, c)
    i += 1


s = "hello"
for i in range(len(s)):
    print(i, s[i])
    i += 1


s = "hello"
for i, c in enumerate(s):
    print(i, c)

# Exception

# try continue/break

a = 0
b = 2

while a < 4:
    print("-" * 30)
    a += 1
    b -= 1

    try:
        _ = a / b
    except ZeroDivisionError:
        print(f"{a}, {b} - division by 0")
        continue
    finally:
        print(f"{a}, {b} - always executes")

    print(f"{a}, {b} - main loop")
    

# ------------------------------
# 1, 1 - always executes
# 1, 1 - main loop
# ------------------------------       
# 2, 0 - division by 0
# 2, 0 - always executes
# ------------------------------       
# 3, -1 - always executes
# 3, -1 - main loop
# ------------------------------       
# 4, -2 - always executes
# 4, -2 - main loop


a = 0
b = 2

while a < 4:
    print("-" * 30)
    a += 1
    b -= 1

    try:
        a / b  # type: ignore
    except ZeroDivisionError:
        print(f"{a}, {b} - division by 0")
        break
    finally:
        print(f"{a}, {b} - always executes")

    print(f"{a}, {b} - main loop")
else:
    print("Code executed without a zero division error")

# ------------------------------
# 1, 1 - always executes
# 1, 1 - main loop
# ------------------------------       
# 2, 0 - division by 0
# 2, 0 - always executes

### NotImplementedError

def scrape_website(url: str) -> dict:
    pass

scrape_website("www.abc.com") # No Error


def scrape_website(url: str) -> dict:
    raise NotImplementedError("Functionality is missing")

scrape_website("www.abc.com") # NotImplementedError: Functionality is missing

# TypeError and ValueError

print(int("10")) # 10

# int can accept string
print(int("x")) # ValueError: invalid literal for int() with base 10: 'x'

# int can't accept None
print(int(None)) # TypeError: int() argument must be a string, a bytes-like object or a real number, not 'NoneType'

# Error to file

import sys
from pprint import pprint


class Person:
    def __del__(self):
        raise ValueError("Something went bump...")


class ErrToFile:
    def __init__(self, fname):
        self._fname = fname
        self._current_stderr = sys.stderr

    def __enter__(self):
        self._file = open(self._fname, "w")
        sys.stderr = self._file

    def __exit__(self, exc_type, exc_value, exc_tb):
        sys.stderr = self._current_stderr
        if self._file:
            self._file.close()
        return False


p = Person()

with ErrToFile("err.txt"):
    del p

with open("err.txt") as f:
    pprint(f.readlines())
    
import sys
from pprint import pprint


class Person:
    def __del__(self):
        raise ValueError("Something went bump...")


class ErrToFile:
    def __init__(self, fname):
        self._fname = fname
        self._current_stderr = sys.stderr

    def __enter__(self):
        self._file = open(self._fname, "w")
        sys.stderr = self._file

    def __exit__(self, exc_type, exc_value, exc_tb):
        sys.stderr = self._current_stderr
        if self._file:
            self._file.close()
        return False


p = Person()

with ErrToFile("err.txt"):
    del p

with open("err.txt") as f:
    pprint(f.readlines())


# [
#     "Exception ignored in: <function Person.__del__ at 0x000001C3684C8860>\n",
#     "Traceback (most recent call last):\n",
#     '  File "c:\\Users\\Babb\\Desktop\\STUDY\\py-deep-dive\\test.py", line '
#     "7, in __del__\n",
#     '    raise ValueError("Something went bump...")\n',
#     "ValueError: Something went bump...\n",
# ]

# EAFP Easier to ask for forgiveness than permission

def get_item_forgive_me(seq, idx, default=None):
    try:
        return seq[idx]
    except (IndexError, TypeError, KeyError):
        # catch either not indexable (TypeError), or index out of bounds,
        # or even a KeyError for mapping types
        return default


def get_item_ask_perm(seq, idx, default=None):
    if hasattr(seq, "__getitem__"):
        if idx < len(seq):
            return seq[idx]
    return default


print(get_item_forgive_me([1, 2, 3], 0))  # 1
print(get_item_forgive_me([1, 2, 3], 10, "Nope"))  # Nope

print(get_item_ask_perm([1, 2, 3], 0))  # 1
print(get_item_ask_perm([1, 2, 3], 10, "Nope"))  # Nope


print(get_item_forgive_me({"a": 100}, "a"))  # 100
# print(get_item_ask_perm({"a": 1}, "a")) # TypeError: '<' not supported between instances of 'str' and 'int'


def get_item_ask_perm(seq, idx, default=None):
    if hasattr(seq, "__getitem__"):
        # could be sequence type or mapping type, or something else altogether??
        if isinstance(seq, dict):
            return seq.get(idx, default)
        elif isinstance(idx, int):
            # looks like a numerical index...
            if idx < len(seq):
                return seq[idx]
    return default


print(get_item_ask_perm({"a": 100}, "a"))  # 100


class ConstantSequence:
    def __init__(self, val):
        self.val = val

    def __getitem__(self, idx):
        return self.val


seq = ConstantSequence(10)
print(seq[0])  # 10


print(get_item_forgive_me(seq, 0, "Nope"))  # 10
# print(get_item_ask_perm(seq, 0, "Nope")) # TypeError: object of type 'ConstantSequence' has no len()

# raise exception from ignore trace back

# try:
#     raise ValueError("level 1")
# except ValueError:
#     try:
#         raise TypeError("level 2")
#     except TypeError:
#         raise KeyError("level 3")

# ValueError: level 1

# During handling of the above exception, another exception occurred:

# ...
# TypeError: level 2

# During handling of the above exception, another exception occurred:

# ...
# KeyError: 'level 3'


# try:
#     raise ValueError("level 1")
# except ValueError:
#     try:
#         raise TypeError("level 2")
#     except TypeError:
#         raise KeyError("level 3") from None


# KeyError: "level 3"


try:
    raise ValueError("level 1")
except ValueError as ex:
    try:
        raise TypeError("level 2")
    except TypeError:
        raise KeyError("level 3") from ex


# ValueError: level 1

# The above exception was the direct cause of the following exception:

# ...
# KeyError: 'level 3'

# API Exception

import json
from datetime import datetime, timezone
from http import HTTPStatus


class APIException(Exception):
    """Base API exception"""

    http_status = HTTPStatus.INTERNAL_SERVER_ERROR
    internal_err_msg = "API exception occurred."
    user_err_msg = "We are sorry. An unexpected error occurred on our end."

    def __init__(self, *args, user_err_msg=None):
        if args:
            self.internal_err_msg = args[0]
            super().__init__(*args)
        else:
            super().__init__(self.internal_err_msg)

        if user_err_msg is not None:
            self.user_err_msg = user_err_msg

    def to_json(self):
        err_object = {"status": self.http_status, "message": self.user_err_msg}
        return json.dumps(err_object)

    def log_exception(self):
        exception = {
            "type": type(self).__name__,
            "http_status": self.http_status,
            "message": self.args[0] if self.args else self.internal_err_msg,
            "args": self.args[1:],
        }
        print(f"EXCEPTION: {datetime.now(timezone.utc).isoformat()}: {exception}")


try:
    raise APIException()
except APIException as ex:
    ex.log_exception()
    print(ex.to_json())
# EXCEPTION: 2025-12-26T03:14:39.060702+00:00: {'type': 'APIException', 'http_status': <HTTPStatus.INTERNAL_SERVER_ERROR: 500>, 'message': 'API exception occurred.', 'args': ()}
# {"status": 500, "message": "We are sorry. An unexpected error occurred on our end."}


class ApplicationException(APIException):
    """Indicates an application error (not user caused) - 5xx HTTP type errors"""

    http_status = HTTPStatus.INTERNAL_SERVER_ERROR
    internal_err_msg = "Generic server side exception."
    user_err_msg = "We are sorry. An unexpected error occurred on our end."


class DBException(ApplicationException):
    """General database exception"""

    http_status = HTTPStatus.INTERNAL_SERVER_ERROR
    internal_err_msg = "Database exception."
    user_err_msg = "We are sorry. An unexpected error occurred on our end."


class DBConnectionError(DBException):
    """Indicates an error connecting to database"""

    http_status = HTTPStatus.INTERNAL_SERVER_ERROR
    internal_err_msg = "DB connection error."
    user_err_msg = "We are sorry. An unexpected error occurred on our end."


class ClientException(APIException):
    """Indicates exception that was caused by user, not an internal error"""

    http_status = HTTPStatus.BAD_REQUEST
    internal_err_msg = "Client submitted bad request."
    user_err_msg = "A bad request was received."


class NotFoundError(ClientException):
    """Indicates resource was not found"""

    http_status = HTTPStatus.NOT_FOUND
    internal_err_msg = "Resource was not found."
    user_err_msg = "Requested resource was not found."


class Account:
    def __init__(self, account_id, account_type):
        self.account_id = account_id
        self.account_type = account_type


class NotAuthorizedError(ClientException):
    """User is not authorized to perform requested action on resource"""

    http_status = HTTPStatus.UNAUTHORIZED
    internal_err_msg = "Client not authorized to perform operation."
    user_err_msg = "You are not authorized to perform this request."


def lookup_account_by_id(account_id):
    # mock of various exceptions that could be raised getting an account from database
    if not isinstance(account_id, int) or account_id <= 0:
        raise ClientException(
            f"Account number {account_id} is invalid.",
            f"account_id = {account_id}",
            "type error - account number not an integer",
        )

    if account_id < 100:
        raise DBConnectionError("Permanent failure connecting to database.", "db=db01")
    elif account_id < 200:
        raise NotAuthorizedError(
            "User does not have permissions to read this account",
            f"account_id={account_id}",
        )
    elif account_id < 300:
        raise NotFoundError("Account not found.", f"account_id={account_id}")
    else:
        return Account(account_id, "Savings")


def get_account(account_id):
    try:
        account = lookup_account_by_id(account_id)
    except APIException as ex:
        ex.log_exception()
        return ex.to_json()
    else:
        return HTTPStatus.OK, {"id": account.account_id, "type": account.account_type}


get_account("ABC")
# EXCEPTION: 2025-12-26T03:19:05.542944+00:00: {'type': 'ClientException', 'http_status': <HTTPStatus.BAD_REQUEST: 400>, 'message': 'Account number ABC is invalid.', 'args': ('account_id = ABC', 'type error - account number not an integer')}
print(get_account(400))  # (<HTTPStatus.OK: 200>, {'id': 400, 'type': 'Savings'})

# Thread and Coroutine

# asyncio.to_thread change sync to async

import asyncio
from asyncio import Task
import requests
from requests import Response

async def fetch_status(url: str) -> dict:
    print(f"Fetch status for: {url}")
    # Asynchronously run function `func` in a separate thread
    # Return a coroutine that can be awaited to get the eventual result of func.
    response: Response = await asyncio.to_thread(requests.get, url, None) 
    print("Done!")
    return {"status": response.status_code, "url": url}

async def main() -> None:
    task_a: Task[dict] = asyncio.create_task(fetch_status("https://fanyi.sogou.com/"))
    task_b: Task[dict] = asyncio.create_task(fetch_status("https://www.youdao.com/"))

    task_a_status: dict = await task_a
    task_b_status: dict = await task_b

    print(task_a_status)
    print(task_b_status)


asyncio.run(main=main())

# Fetch status for: https://fanyi.sogou.com/
# Fetch status for: https://www.youdao.com/
# Done!
# Done!
# {'status': 200, 'url': 'https://fanyi.sogou.com/'}
# {'status': 200, 'url': 'https://www.youdao.com/'}

# Regex

# re.search() and re.match()

re.search() 查找第一个符合条件的，不要求必须是文字开头。
re.match() 查找文字开头是否符合条件。

import re

string = 'abc1de2f'

m = re.search('[0-9]', string)
print(m, m.group()) # <_sre.SRE_Match object; span=(3, 4), match='1'>  1

m = re.match('[0-9]', string)
print(m) # None

m = re.match('[0-9]', '1abc1de2f')
print(m, m.group()) # <_sre.SRE_Match object; span=(0, 1), match='1'>  1

# name group

name group: 命名组，可以通过 ?P<name> 给查找到的 group 命名

import re

m = re.search(r'out_(\d{4})','out_1984.txt')
print(m.group(0), m.group(1)) # out_1984 1984

m = re.search(r'out_(?P<year>\d{4})', 'out_1984.txt')
print(m.group('year')) # 1984

# reference group

reference group: 引用组，可以通过 \n 引用分组，代表引用第 n 个分组。

import re

# \1 表示引用第一个分组
m = re.match(r"<([a-zA-Z]*)>\w*</\1>", "<div>Hello</div>")
print(m.group(), m.group(0), m.group(1)) # <div>Hello</div> <div>Hello</div> div

# operate sub group

import re

s = "start TT end"
res = re.sub(r"([A-Z])\1", lambda pat: pat.group(1).lower(), s)
print(res)  # start t end

# regex example

提取网页中的文字内容。

import re

data_str = """
<div>
        <p>岗位职责：</p>
<p>完成推荐算法、数据统计、接口、后台等服务器端相关工作</p>
</div>
"""

# [^>] 非 > 字符
print(re.sub(r"<[^>]*>|&nbsp;|\n", "", data_str)) # 岗位职责：完成推荐算法、数据统计、接口、后台等服务器端相关工作

# .*? 匹配任意字符一个或多个, ? 非贪婪匹配
print(re.sub(r"<.*?>|&nbsp;|\n", "", data_str)) # 岗位职责：完成推荐算法、数据统计、接口、后台等服务器端相关工作

# Web

# html_request_header to dict

将请求头通过分割转换为列表，再通过 dict(list) 函数将 list 转换为 dict.
dict([['Host', 'open.tool.hexun.com']]) => {'Host': 'open.tool.hexun.com'}

raw_headers = """Host: open.tool.hexun.com
Pragma: no-cache
Cache-Control: no-cache
User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36
Accept: */*
Referer: http://stock.hexun.com/gsxw/
Accept-Encoding: gzip, deflate
Accept-Language: zh-CN,zh;q=0.9,en;q=0.8"""

headers = dict([line.split(": ") for line in raw_headers.split("\n")])
print(headers)

# {'Host': 'open.tool.hexun.com', 'Pragma': 'no-cache', 'Cache-Control': 'no-cache', 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36', 'Accept': '*/*', 'Referer': 'http://stock.hexun.com/gsxw/', 'Accept-Encoding': 'gzip, deflate', 'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8'}

# request_html

request_html 将获取页面和解析页面内容整合在一起。

from requests_html import HTMLSession

session = HTMLSession()
r = session.get('https://python.org/')
print(r.html.links)

about = r.html.find('#about', first=True)
print(about.text)

content = r.html.search('python is a {} language')[0]
print(content)

# HTMLParser unescape from HTML Entity

将 HTML Entity 转换为普通字符

from html.parser import HTMLParser

def decode_html(input):
    h = HTMLParser()
    s = h.unescape(input)
    return s

print(decode_html('and#38451;&#38175;&#12345;&lt;&sect;')) # 阳锟〹<§

# pyppeteer

import asyncio
from pyppeteer import launch


async def main():
    browser = await launch()
    page = await browser.newPage()
    await page.goto('http://example.com')
    await page.screenshot({'path': 'example.png'})
    await browser.close()

# AttributeError: module 'asyncio' has no attribute 'run'
# use python 3.7
# asyncio.run(main())

loop = asyncio.get_event_loop()
loop.run_until_complete(asyncio.wait([main()]))
loop.close()

# flask index page

@app.route('/', methods = ['GET', 'POST'])
@app.route('/page=<page>', methods = ['GET', 'POST'])
def index(page=1):
    pass

# Other

# python and python -m

python run.py 是把 run.py 所在的目录放到 sys.path 属性中。

python -m run.py 是把输入命令的目录放到 sys.path 属性中。

# venv

python3 create virtual enviroment.

创建虚拟环境 python -m venv my-project
激活虚拟环境
在 Posix 标准平台下： source <venv>/bin/activate
- 在 Windows 下： <venv>/Scripts/activate.bat

cmd 进入需要共享的文件夹，执行 python -m http.server
ipconfig/all 查看电脑 ip
手机端连接和电脑同一个网段的 wifi，在浏览器输入电脑 ip:port 即可共享文件

# python cProfile

From: Analyzing Slow python Code using cProfile

from time import sleep
import cProfile
import pstats


profiler = cProfile.Profile()
profiler.enable()


def a():
    sleep(1)
    b()
    sleep(1)


def b():
    sleep(1)


a()

profiler.disable()
stats = pstats.Stats(profiler)
stats.dump_stats("./cProfile.stats")
stats.print_stats()

# $ python test.py
#          6 function calls in 3.002 seconds

#    Random listing order was used

#    ncalls  tottime  percall  cumtime  percall filename:lineno(function)
#         1    0.000    0.000    1.001    1.001 C:\Users\Babb\Desktop\STUDY\cprofile-test\test.py:16(b)
#         1    0.000    0.000    3.002    3.002 C:\Users\Babb\Desktop\STUDY\cprofile-test\test.py:10(a)
#         3    3.002    1.001    3.002    1.001 {built-in method time.sleep}
#         1    0.000    0.000    0.000    0.000 {method 'disable' of '_lsprof.Profiler' objects}

$ pip install snakeviz

snakeviz cProfile.stats

# chardet

根据文件内容检测编码。

with open("chardet_test.py", "rb") as f:
    file_data = f.read()
    result = chardet.detect(file_data)
    print(result)

    file_content = file_data.decode(encoding=result["encoding"])
    print("-" * 50)
    print(file_content)

# delete bom

1
2
3

s = open('./850_with_bom.json', mode='r', encoding='utf-8-sig').read()

open('./850.json', mode='w', encoding='utf-8').write(s)

# windows pip install PermissionError

PermissionError: [WinError 5] 拒绝访问。

找到 python 安装目录赋予当前用户完全控制的权限。

# windows schedule call python

如果没有设置 python 环境变量可以在开始位置写入 python.exe 的路径，如 C:\Program Files\python36

# python run daemon on window

"C:\Program Files\python36\pythonw.exe" test.py
# OR
"C:\Program Files\python36\pythonw.exe" test.py >log.txt 2>&1
# OR
"C:\Program Files\python36\pythonw.exe" test.py 1>stdout.txt 2>stderr.txt

# replace double slash to one slash

s = r'foo \\ bar'
print(s)
print(s.replace('\\\\', '\\'))

s = 'foo \\\\ bar'
print(s)
print(s.replace('\\\\', '\\'))

# foo \\ bar
# foo \ bar
# foo \\ bar
# foo \ bar

# replace normal unicode str to unicode encoded str

将普通的 Unicode 字符串转换为 Unicode 编码的字符串。
'\\u9500\\u552e' or r'\u9500\u552e'=> '\u9500\u552e'

import json
import re

s = "\\u9500\\u552e"
print(s) 			  # \u9500\u552e
print("\u9500\u552e") # 销售

# SyntaxError: (unicode error) 'unicodeescape' codec can't decode bytes in position 0-1: truncated \uXXXX escape
# \u 不能组成完整字符
# res = re.sub('\\u', '\u', s)


s = "\\u9500\\u552e"
print(json.loads(f'"{s}"')) # 销售

print(s.encode().decode("unicode_escape")) # 销售

# while loop break

# command = ''
# while command.lower() != 'quit':
#     command = input('>')
#     result = eval(command)
#     print(result)


while True:
    command = input('>')
    print('ECHO', command)
    if command.lower() == 'quit':
        break

# douban pypi

1	pip install requests -i https://pypi.douban.com/simple/

# read toml

import toml
from pprint import pprint as print


def read_toml(file_path):
    with open(file_path, "r") as f:
        data = toml.load(f)
    return data


print(read_toml("./config.toml"))

config.toml

# Even Better TOML
name = 'Babb'
age = 50
sex = "\"Male\""
hobbies = ['reading', 'coding', 'gaming']
birthdate = 1998-07-07T00:00:00Z
shape = { height = 170, wiight=60 }

# [account]
# bilibili.username = 'babb'
# bilibili.password = 'xxx'
# youtube.username = 'babb'
# youtube.password = 'xxx'


# [account.bilibili]
# username = 'babb'
# password = 'xxx'

# [account.youtube]
# username = 'babb'
# password = 'xxx'


# account = [
#     {name = 'bilibili', username = 'babb', password='xxx'},
#     {name = 'youtube', username = 'babb', password='xxx'}
# ]

[[account]]
name = 'bilibili'
username = 'babb'
password='xxx'


[[account]]
name = 'youtube'
username = 'babb'
password='xxx'


# {'account': [{'name': 'bilibili', 'password': 'xxx', 'username': 'babb'},
#              {'name': 'youtube', 'password': 'xxx', 'username': 'babb'}],
#  'age': 50,
#  'birthdate': datetime.datetime(1998, 7, 7, 0, 0, tzinfo=<toml.tz.TomlTz object at 0x000001F95DC7B8C0>),
#  'hobbies': ['reading', 'coding', 'gaming'],
#  'name': 'Babb',
#  'sex': '"Male"',
#  'shape': {'height': 170, 'wiight': 60}}

# json serialize

import json


d = {"a": float("inf"), "b": float("nan")}
s = json.dumps(d)
print(s)  # {"a": Infinity, "b": NaN}

# s = json.dumps(d, allow_nan=False)  # ValueError: Out of range float values are not JSON compliant: inf


d = {10: "int", 10.5: "float", (1, 1): "complex"}
# s = json.dumps(d)  # TypeError: keys must be str, int, float, bool or None, not tuple

s = json.dumps(d, skipkeys=True)
print(s)  # {"10": "int", "10.5": "float"}


d = {"name": "Python", "age": 27, "created_by": "Guido van Rossum", "list": [1, 2, 3]}
s = json.dumps(d)
print(s)
# {"name": "Python", "age": 27, "created_by": "Guido van Rossum", "list": [1, 2, 3]}

s = json.dumps(d, indent="--", separators=(";", " = "))
print(s)
# {
# --"name" = "Python";
# --"age" = 27;
# --"created_by" = "Guido van Rossum";
# --"list" = [
# ----1;
# ----2;
# ----3
# --]
# }

from datetime import datetime, UTC
import json


class Person:
    def __init__(self, name, age):
        self.name = name
        self.age = age

    def __repr__(self):
        return f"Person(name={self.name}, age={self.age})"

    def toJSON(self):
        # return {
        #     'name': self.name,
        #     'age': self.age,
        # }
        return vars(self)


class Point:
    def __init__(self, x, y):
        self.x = x
        self.y = y

    def __repr__(self):
        return f"Point(x={self.x}, y={self.y})"


def custom_json_formatter(arg):
    if isinstance(arg, datetime):
        return arg.isoformat()
    elif isinstance(arg, set):
        return list(arg)
    else:
        try:
            return arg.toJSON()
        except AttributeError:
            try:
                return vars(arg)
            except TypeError:
                return str(arg)


person = Person("John", 82)

point = Point(10, 10)

log_record = dict(
    message="Create new point",
    point=point,
    created_time=datetime.now(UTC),
    created_by=person,
)
print(json.dumps(log_record, default=custom_json_formatter, indent=2))

# {
#   "message": "Create new point",
#   "point": {
#     "x": 10,
#     "y": 10
#   },
#   "created_time": "2025-12-25T05:14:33.243506+00:00",
#   "created_by": {
#     "name": "John",
#     "age": 82
#   }
# }

import json
from decimal import Decimal
from fractions import Fraction
from datetime import datetime, UTC
from functools import singledispatch


class Person:
    def __init__(self, name, age):
        self.name = name
        self.age = age

    def __repr__(self):
        return f"Person(name={self.name}, age={self.age})"

    def toJSON(self):
        # return {
        #     'name': self.name,
        #     'age': self.age,
        # }
        return vars(self)


class Point:
    def __init__(self, x, y):
        self.x = x
        self.y = y

    def __repr__(self):
        return f"Point(x={self.x}, y={self.y})"


@singledispatch
def json_format(arg):
    print(arg)
    try:
        print("\ttrying to use toJSON()...")
        return arg.toJSON()
    except AttributeError:
        print("\tfailed - trying to use vars...")
        try:
            return vars(arg)
        except TypeError:
            print("\tfailed - using string repr...")
            return str(arg)


@json_format.register(datetime)
def _(arg):
    return arg.isoformat()


@json_format.register(set)
def _(arg):
    return list(arg)


person = Person("John", 82)

point = Point(10, 10)

log_record = dict(
    message="Create new point",
    point=point,
    created_time=datetime.now(UTC),
    created_by=person,
)
print(json.dumps(log_record, default=json_format, indent=2))


# Point(x=10, y=10)
#         trying to use toJSON()...
#         failed - trying to use vars...
# Person(name=John, age=82)
#         trying to use toJSON()...
# {
#   "message": "Create new point",
#   "point": {
#     "x": 10,
#     "y": 10
#   },
#   "created_time": "2025-12-25T05:18:19.629203+00:00",
#   "created_by": {
#     "name": "John",
#     "age": 82
#   }
# }


@json_format.register(Decimal)
def _(arg):
    return f"Decimal({str(arg)})"


result = json.dumps(
    dict(
        a=1 + 1j,
        b=Decimal("0.5"),
        c=Fraction(1, 3),
        p=Person("Python", 27),
        pt=Point(0, 0),
        time=datetime.now(UTC),
    ),
    default=json_format,
    indent=2,
)
print(result)

# (1+1j)
#         trying to use toJSON()...
#         failed - trying to use vars...
#         failed - using string repr...
# 1/3
#         trying to use toJSON()...
#         failed - trying to use vars...
#         failed - using string repr...
# Person(name=Python, age=27)
#         trying to use toJSON()...
# Point(x=0, y=0)
#         trying to use toJSON()...
#         failed - trying to use vars...
# {
#   "a": "(1+1j)",
#   "b": "Decimal(0.5)",
#   "c": "1/3",
#   "p": {
#     "name": "Python",
#     "age": 27
#   },
#   "pt": {
#     "x": 0,
#     "y": 0
#   },
#   "time": "2025-12-25T05:20:48.137975+00:00"
# }

import json
from datetime import datetime, UTC


class CustomEncoder(json.JSONEncoder):
    def __init__(self, *args, **kwargs):
        super().__init__(skipkeys=True, allow_nan=False, indent=2)

    def default(self, arg):
        if isinstance(arg, datetime):
            return dict(
                datatype="datetime",
                iso=arg.isoformat(),
                date=arg.date().isoformat(),
                time=arg.time().isoformat(),
                year=arg.year,
                month=arg.month,
                day=arg.day,
                hour=arg.hour,
                minutes=arg.minute,
                seconds=arg.second,
            )
        else:
            return super().default(arg)


d = {"time": datetime.now(UTC), 1 + 1j: "complex", "name": "python"}
s = json.dumps(d, cls=CustomEncoder)
print(s)

# {
#     "time": {
#         "datatype": "datetime",
#         "iso": "2025-12-25T05:35:28.154665+00:00",
#         "date": "2025-12-25",
#         "time": "05:35:28.154665",
#         "year": 2025,
#         "month": 12,
#         "day": 25,
#         "hour": 5,
#         "minutes": 35,
#         "seconds": 28,
#     },
#     "name": "python",
# }

serialize date

import json
from json import JSONEncoder
from datetime import datetime
from datetime import date

now = datetime.now()

# TypeError: Object of type datetime is not JSON serializable
# print(json.dumps(now))

class CustomEncoder(JSONEncoder):
    def default(self, obj):

        if isinstance(obj, date) or isinstance(obj, datetime):
            return obj.isoformat()

        return super().default(obj)

print(json.dumps(now, cls=CustomEncoder)) # "2024-09-24T15:43:34.821884"

serialize custom object

class Person:
    def __init__(self, name, born):
        self.name = name
        self.born = born

    @property
    def age(self):
        return datetime.now().year - self.born

person = Person("Babb Chen", 1994)
# TypeError: Object of type Person is not JSON serializable
# print(json.dumps(person))

# Not include property age
print(person.__dict__) # {'name': 'Babb Chen', 'born': 1994}
print(json.dumps(person.__dict__)) # {"name": "Babb Chen", "born": 1994}

def seriealize_person(obj):
    if isinstance(obj, Person):
        return {
            "name": obj.name,
            "age": obj.age
        }

    raise TypeError("Object not serializable")

print(json.dumps(person, default=seriealize_person)) # {"name": "Babb Chen", "age": 30}

# json deserialize

from datetime import datetime
from fractions import Fraction
import json
from pprint import pprint


def custom_decoder(arg):
    print("decoding:", arg, type(arg))
    return arg


s = """
    {
        "a": 1,
        "b": 2, 
        "c": {
            "c.1": 1,
            "c.2": 2,
            "c.3": {
                "c.3.1": 1,
                "c.3.2": 2
            }
        }
    }
"""

d = json.loads(s, object_hook=custom_decoder)
print(d)
# decoding: {'c.3.1': 1, 'c.3.2': 2} <class 'dict'>
# decoding: {'c.1': 1, 'c.2': 2, 'c.3': {'c.3.1': 1, 'c.3.2': 2}} <class 'dict'>
# decoding: {'a': 1, 'b': 2, 'c': {'c.1': 1, 'c.2': 2, 'c.3': {'c.3.1': 1, 'c.3.2': 2}}} <class 'dict'>
# {'a': 1, 'b': 2, 'c': {'c.1': 1, 'c.2': 2, 'c.3': {'c.3.1': 1, 'c.3.2': 2}}}


def custom_decoder(arg):
    ret_value = arg
    if "objecttype" in arg:
        if arg["objecttype"] == "datetime":
            ret_value = datetime.strptime(arg["value"], "%Y-%m-%dT%H:%M:%S")
        elif arg["objecttype"] == "fraction":
            ret_value = Fraction(arg["numerator"], arg["denominator"])
    return ret_value


s = """
    {
        "cake": "yummy chocolate cake",
        "myShare": {
            "objecttype": "fraction",
            "numerator": 1,
            "denominator": 8
        },
        "eaten": {
            "at": {
                "objecttype": "datetime",
                "value": "2018-10-21T21:30:00"
                },
            "time_taken": "30 seconds"
        }
    }
"""

d = json.loads(s, object_hook=custom_decoder)
pprint(d)
# {
#     "cake": "yummy chocolate cake",
#     "eaten": {
#         "at": datetime.datetime(2018, 10, 21, 21, 30),
#         "time_taken": "30 seconds",
#     },
#     "myShare": Fraction(1, 8),
# }

import json


def float_handler(arg):
    print("float handler", type(arg), arg)
    return float(arg)


def int_handler(arg):
    print("int handler", type(arg), arg)
    return int(arg)


def const_handler(arg):
    print("const handler", type(arg), arg)
    return None


def obj_hook(arg):
    print("obj hook", type(arg), arg)
    return arg


def obj_pairs_hook(arg):
    print("obj pairs hook", type(arg), arg)
    return arg


s = """
    {
        "a": [1, 2, 3, 4, 5],
        "b": 100,
        "c": 10.5,
        "d": NaN,
        "e": null,
        "f": "python"
    }
"""


d = json.loads(
    s,
    object_hook=obj_hook,
    parse_float=float_handler,
    parse_int=int_handler,
    parse_constant=const_handler,
)
print(d)
# int handler <class 'str'> 1
# int handler <class 'str'> 2
# int handler <class 'str'> 3
# int handler <class 'str'> 4
# int handler <class 'str'> 5
# int handler <class 'str'> 100
# float handler <class 'str'> 10.5
# const handler <class 'str'> NaN
# obj hook <class 'dict'> {'a': [1, 2, 3, 4, 5], 'b': 100, 'c': 10.5, 'd': None, 'e': None, 'f': 'python'}
# {'a': [1, 2, 3, 4, 5], 'b': 100, 'c': 10.5, 'd': None, 'e': None, 'f': 'python'}


d = json.loads(
    s,
    object_pairs_hook=obj_pairs_hook,
    parse_float=float_handler,
    parse_int=int_handler,
    parse_constant=const_handler,
)
print(d)

# int handler <class 'str'> 1
# int handler <class 'str'> 2
# int handler <class 'str'> 3
# int handler <class 'str'> 4
# int handler <class 'str'> 5
# int handler <class 'str'> 100
# float handler <class 'str'> 10.5
# const handler <class 'str'> NaN
# obj pairs hook <class 'list'> [('a', [1, 2, 3, 4, 5]), ('b', 100), ('c', 10.5), ('d', None), ('e', None), ('f', 'python')]
# [('a', [1, 2, 3, 4, 5]), ('b', 100), ('c', 10.5), ('d', None), ('e', None), ('f', 'python')]

import re
import json
from decimal import Decimal
from pprint import pprint


class Point:
    def __init__(self, x, y):
        self.x = x
        self.y = y

    def __repr__(self):
        return f"Point(x={self.x}, y={self.y})"


s = """
{
    "a": 100,
    "b": 0.5,
    "rectangle": {
        "corners": {
            "b_left": {"_type": "point", "x": -1, "y": -1},
            "b_right": {"_type": "point", "x": 1, "y": -1},
            "t_left": {"_type": "point", "x": -1, "y": 1},
            "t_right": {"_type": "point", "x": 1, "y": 1}
        },
        "rotate": {"_type" : "point", "x": 0, "y": 0},
        "interior_pts": [
            {"_type": "point", "x": 0, "y": 0},
            {"_type": "point", "x": 0.5, "y": 0.5}
        ]
    }
}
"""


class CustomDecoder(json.JSONDecoder):
    base_decoder = json.JSONDecoder(parse_float=Decimal)

    def decode(self, s):
        obj = self.base_decoder.decode(s)
        pattern = r'"_type"\s*:\s*"point"'
        if re.search(pattern, s):
            obj = self.make_pts(obj)
        return obj

    def make_pts(self, obj):
        if isinstance(obj, dict):
            if "_type" in obj and obj["_type"] == "point":
                obj = Point(obj["x"], obj["y"])
            else:
                for key, value in obj.items():
                    obj[key] = self.make_pts(value)
        elif isinstance(obj, list):
            for index, item in enumerate(obj):
                obj[index] = self.make_pts(item)

        return obj


d = json.loads(s, cls=CustomDecoder)
pprint(d)
# {
#     "a": 100,
#     "b": Decimal("0.5"),
#     "rectangle": {
#         "corners": {
#             "b_left": Point(x=-1, y=-1),
#             "b_right": Point(x=1, y=-1),
#             "t_left": Point(x=-1, y=1),
#             "t_right": Point(x=1, y=1),
#         },
#         "interior_pts": [Point(x=0, y=0), Point(x=0.5, y=0.5)],
#         "rotate": Point(x=0, y=0),
#     },
# }


pts = d["rectangle"]["interior_pts"][1]
print(type(pts.x), type(pts.y))  # <class 'decimal.Decimal'> <class 'decimal.Decimal'>

# jsonschema

import json
import jsonschema


person_schema = {
    "type": "object",
    "properties": {
        "firstName": {"type": "string", "minLength": 1},
        "middleInitial": {"type": "string", "minLength": 1, "maxLength": 1},
        "lastName": {"type": "string", "minLength": 1},
        "age": {"type": "integer", "minimum": 0},
        "eyeColor": {
            "type": "string",
            "enum": [
                "amber",
                "blue",
                "brown",
                "gray",
                "green",
                "hazel",
                "red",
                "violet",
            ],
        },
    },
    "required": ["firstName", "lastName"],
}


json_doc = """
    {
        "firstName": "John",
        "middleInitial": 100,
        "lastName": "Cleese",
        "age": "Unknown"
    }
"""

try:
    jsonschema.validate(json.loads(json_doc), person_schema)
except json.JSONDecodeError as e:
    print(f"Invalid JSON: {e}")
except jsonschema.exceptions.ValidationError as e:
    print(f"Validition JSON error: {e}")
else:
    print("JSON is valid and conforms to schema")

# Validition JSON error: 100 is not of type 'string'

# Failed validating 'type' in schema['properties']['middleInitial']:
#     {'type': 'string', 'minLength': 1, 'maxLength': 1}

# On instance['middleInitial']:
#     100


for error in jsonschema.Draft4Validator(person_schema).iter_errors(
    json.loads(json_doc)
):
    print(error, end="\n------------------\n")

# 100 is not of type 'string'

# Failed validating 'type' in schema['properties']['middleInitial']:
#     {'type': 'string', 'minLength': 1, 'maxLength': 1}

# On instance['middleInitial']:
#     100
# ------------------
# 'Unknown' is not of type 'integer'

# Failed validating 'type' in schema['properties']['age']:
#     {'type': 'integer', 'minimum': 0}

# On instance['age']:
#     'Unknown'
# ------------------

# format XML

import xml.dom.minidom as xdom
from pathlib import Path

work_dir = r'C:\Users\11435\Desktop\practice\python\formatxml\xmlfiles'

paths = Path(work_dir).glob('**/*.xml')
for path in paths:
    prettied = xdom.parseString(path.read_text()).toprettyxml()
    path.write_text(pettied)

# nested XML to pandas dataframe

data.xml

<data>
    <record>
        <name>Babb</name>
        <age>30</age>
        <address>
            <city>New York</city>
            <state>NY</state>
        </address>
    </record>
    <record>
        <name>John</name>
        <age>25</age>
        <address>
            <city>New York</city>
            <state>NY</state>
        </address>
    </record>
</data>

import pandas as pd
import xml.etree.ElementTree as ET

tree = ET.parse('data.xml')
root = tree.getroot()

def parse_element(element, item):
    if len(list(element)) == 0:
        item[element.tag] = element.text
    else:
        for child in list(element):
            parse_element(child, item)

data = []

for child in root:
    item = {}
    parse_element(child, item)
    data.append(item)

print(data)
# [{'name': 'Babb', 'age': '30', 'city': 'New York', 'state': 'NY'}, {'name': 'John', 'age': '25', 'city': 'New York', 'state': 'NY'}]

df = pd.DataFrame(data)
print(df)
#    name age      city state
# 0  Babb  30  New York    NY
# 1  John  25  New York    NY

Use pandas.read_xml iterparse

iterparse dict, optional
The nodes or attributes to retrieve in iterparsing of XML document as a dict with key being the name of repeating element and value being list of elements or attribute names that are descendants of the repeated element. Note: If this option is used, it will replace xpath parsing and unlike xpath, descendants do not need to relate to each other but can exist any where in document under the repeating element. This memory-efficient method should be used for very large XML files (500MB, 1GB, or 5GB+). For example, iterparse = {"row_element": ["child_elem", "attr", "grandchild_elem"]

# Define the structure for iterparse
iterparse_config = {
    "record": ["name", "age", "city", "state"]
}

# Read the XML using iterparse
df = pd.read_xml('data.xml', iterparse=iterparse_config)

print(df)
#    name age      city state
# 0  Babb  30  New York    NY
# 1  John  25  New York    NY

from io import BytesIO, StringIO
import pandas as pd

xml_data = '''
<data>
    <record>
        <name>Babb</name>
        <age>30</age>
        <address>
            <city>New York</city>
            <state>NY</state>
        </address>
    </record>
    <record>
        <name>John</name>
        <age>25</age>
        <address>
            <city>New York</city>
            <state>NY</state>
        </address>
    </record>
</data>
'''

# Define the structure for iterparse
iterparse_config = {
    "record": ["name", "age", "city", "state"]
}

# Read the XML using iterparse, default parser is 'lxml'
# df = pd.read_xml(StringIO(xml_data.encode()), iterparse=iterparse_config) # TypeError: reading file objects must return bytes objects
df = pd.read_xml(BytesIO(xml_data.encode()), iterparse=iterparse_config)
# from lxml import etree
# for event, elements in etree.iterparse(BytesIO(xml_data.encode()):
#     pass

# Use parser 'etree'
df = pd.read_xml(BytesIO(xml_data.encode()), iterparse=iterparse_config, parser='etree')
df = pd.read_xml(StringIO(xml_data), iterparse=iterparse_config, parser='etree')

print(df)

# fake large csv

import pandas as pd
from faker import Faker


def generate_large_csv(num_rows, filename):
    fake = Faker()
    data = []
    for _ in range(num_rows):
        data.append(
            [
                fake.uuid4(),
                fake.name(),
                fake.email(),
                fake.date_between(start_date="-5y", end_date="today"),
                fake.country(),
                fake.random_int(min=1000, max=100000),
            ]
        )
    df = pd.DataFrame(data, columns=["id", "name", "email", "date", "country", "value"])
    df.to_csv(filename, index=False)
    print(f"Generated {num_rows} rows and saved to {filename}")


if __name__ == "__main__":
    generate_large_csv(100_000, "large_data.csv")

# csv to database

import pandas as pd
from sqlalchemy import create_engine

# import os
# os.environ["NLS_LANG"] = "GERMAN_GERMANY.UTF8"

# 初始化数据库连接，使用 cx_oracle 模块
engine = create_engine('oracle+cx_oracle://username:pass@ip:port/instance')

# 读取本地 CSV 文件
df = pd.read_csv("test.csv", sep=',')

# 将新建的 DataFrame 储存为 Oracle 中的数据表，不储存 index 列
# 字符串在 Oracle 中存储的是 Clob 对象
df.to_sql('test', engine, index= False)

print("Write to Oracle successfully!")

# quit python Interactive Shell

python Interactive Shell 通过 EOF (End Of File) 退出，python 在 parse stdin 的时候判断是否是 EOF。

Mac/Linux 按 Ctrl + D 退出，Window 先按 Ctrl + Z 然后按 Enter 退出。

1
2
3

python 3.9.6 (tags/v3.9.6:db3ff76, Jun 28 2021, 15:26:21) [MSC v.1929 64 bit (AMD64)] on win32
Type "help", "copyright", "credits" or "license" for more information.
>>> ^Z

# Jupyter notebook to markdown

jupyter nbconvert -h

jupyter nbconvert mynotebook.ipynb --to markdown

# prettytable

from prettytable import PrettyTable
row = PrettyTable(["城市","总访问量", "失败量", "慢速量","失败率","慢速率"])
row.align["城市"] = "l" # Left align city names
row.padding_width = 1   # One space between column edges and contents (default)

retrow = [['郑州',241,1,2,0.41,0.83],['灵宝',56,0,1,0.0,1.79]]
for r in retrow:
    row.add_row(r)
with open('D:/study/fragment/mytable','w',encoding='utf-8') as hfile:
    hfile.write(str(row))
    hfile.close()

# mytable
# +------+----------+--------+--------+--------+--------+
# | 城市 | 总访问量  | 失败量 | 慢速量 | 失败率 | 慢速率 |
# +------+----------+--------+--------+--------+--------+
# | 郑州 |   241    |   1    |   2    |  0.41  |  0.83  |
# | 灵宝 |    56    |   0    |   1    |  0.0   |  1.79  |
# +------+----------+--------+--------+--------+--------+

# tqdm

from tqdm import tqdm


# for _ in tqdm(range(10000)):
#     time.sleep(0.001)

# for _ in trange(10000):
#     time.sleep(0.001)

# for _ in tqdm(range(10000), desc="description"):
#     time.sleep(0.001)


# list1 = ["a", "b", "c", "d", "e"]
# for _ in tqdm(list1):
#     time.sleep(0.5)


# def generator():
#     for i in range(50):
#         yield i


# for _ in tqdm(generator(), total=50):
#     time.sleep(0.1)


# with tqdm(total=100) as progress_bar:
#     progress_bar.update(10)
#     time.sleep(1)
#     progress_bar.update(20)
#     time.sleep(1)
#     progress_bar.update(30)
#     time.sleep(2)
#     progress_bar.update(100)


import seaborn as sns

df = sns.load_dataset("iris")
print(df.head())

# df.petal_width.apply(lambda x: x * 2)

tqdm.pandas(desc="Processing iris")
df.petal_width.progress_apply(lambda x: x * 2)

# dotenv to enviroment

import os
from dotenv import load_dotenv

# load_dotenv("./project_a.env")
load_dotenv()

db_user = os.getenv("DB_USER")
db_password = os.getenv("DB_PASSWORD")
print(f"{db_user=}")
print(f"{db_password=}")

.env

1 2	DB_USER = 'fake_username' DB_PASSWORD = 'fake_password'

# Quiz

# Why judgement None must use ‘is’ in python?

a = None

# 1
if not a:
    print('None')

# 2
if a == None:
    print('None')

# 3
if a is None:
    print('None')

# None
# None
# None

第一种方式在 [], {}, set(), None, False 这几种情况下都符合，用户自定义类型也可以通过重载 __bool__ 定义什么时候是 True/False.

lst = [[], {}, set(), None, False]
for a in lst:
    if not a:
        print(a)

# []
# {}
# set()
# None
# False

class MyDataStructure:
    def __bool__(self):
        return False

d = MyDataStructure()
if not d:
    print("It's None")

# It's None

第二种方式，用户自定义类型可以通过重载 eq 定义什么时候是 True/False.

class MyDataStructure:
    def __eq__(self, other):
        return True

d = MyDataStructure()
if d == None:
    print(d)
# <__main__.MyDataStructure object at 0x000001BAA1B0AFA0>

# Why list comprehensions multi lambdas ouput same result?

funcs = [lambda x: x**i for i in range(6)]

print(funcs[0](10), funcs[1](10), funcs[2](10))
# 100000 100000 100000

Internal Mechanics of List Comprehensions

import dis

compiled_code = compile('[i**2 for i in (1, 2, 3)]', filename='', mode='eval')
dis.dis(compiled_code)

# 0 LOAD_CONST               0 (<code object <listcomp> at 0x000001F77210ED20, file "", line 1>)
# 2 LOAD_CONST               1 ('<listcomp>')
# 4 MAKE_FUNCTION            0
# 6 LOAD_CONST               5 ((1, 2, 3))
# 8 GET_ITER
# 10 CALL_FUNCTION            1
# 12 RETURN_VALUE

As you can see, in step 4, python created a function ( MAKE_FUNCTION ), called it ( CALL_FUNCTION ), and then returned the result ( RETURN_VALUE ) in the last step.

So, comprehensions will behave like functions in terms of scope. They have local scope, and can access global and nonlocal scopes too. And nested comprehensions will also behave like nested functions and closures.

In funcs = [lambda x: x**i for i in range(6)] the i was a local variable in the list comprehension, and each function created in the comprehension is referencing the same i - it is local to the comprehension, and each lambda is therefore a closure with (the same) free variable i . And by the time the comprehension has finished running, i had a value of 5

The above code is like below, except the i is global variable.

funcs = []
for i in range(6):
    funcs.append(lambda x: x**i)

print(i, 'i' in globals())
print(funcs[0](10), funcs[1](10), funcs[2](10))
# 5 True
# 100000 100000 100000

Can we somehow fix this problem?

Yes, and it relies on default values and when default values are calculated and stored with the function definition. Recall thatd efault values are evaluated and stored with the function’s definition when the function is being created (i.e. compiled). Right now we are running into a problem because the free variable i is being evauated inside each function’s body at run time.

So, we can fix this by making each current value of i a paramer default of each lambda - this will get evaluated at the functions creation time - i.e. at each loop iteration:

funcs = [lambda x, pow=i: x**pow for i in range(6)]

print(funcs[0](10), funcs[1](10), funcs[2](10))

# 1, 10, 100

# The difference of iterable and iterator?

iterator must be iterable，but iterable not must be iterator.

# iterator

implement __iter__ and __next__ method.

class Cities:
    def __init__(self):
        self._cities = ['Paris', 'Berlin', 'Rome', 'Madrid', 'London']
        self._index = 0

    def __iter__(self):
        return self

    def __next__(self):
        if self._index >= len(self._cities):
            raise StopIteration
        else:
            item = self._cities[self._index]
            self._index += 1
            return item

yield generator

import math

def factorials(n):
    for i in range(n):
        yield math.factorial(i)

for num in factorials(5):
    print(num)

# 1
# 1
# 2
# 6
# 24

f = factorials(5)
print('__iter__' in dir(f), '__next__' in dir(f)) # True True

# iterable

implement __getitem__ method

class Fib:
    def __init__(self, n):
        self._n = n

    def __len__(self):
        return self._n

    def __getitem__(self, s):
        if isinstance(s, int):
            # single item requested
            if s < 0:
                s = self._n + s
            if s < 0 or s > self._n - 1:
                raise IndexError
            return self._fib(s)
        else:
            # slice being requested
            idx = s.indices(self._n)
            rng = range(idx[0], idx[1], idx[2])
            return [self._fib(n) for n in rng]

    @staticmethod
    @lru_cache(2**32)
    def _fib(n):
        if n < 2:
            return 1
        else:
            return fib(n-1) + fib(n-2)
f = Fib(10)

for item in f:
    print(item)
# 1
# 1
# 2
# 3
# 5
# 8
# 13
# 21
# 34
# 55

# print(f[5::-1]) # [8, 5, 3, 2, 1, 1]

implement __iter__ method and the method return iterator, but not implement __next__ .

class Cities:
    def __init__(self):
        self._cities = ['New York', 'Newark', 'New Delhi', 'Newcastle']

    def __len__(self):
        return len(self._cities)

    def __getitem__(self, s):
        print('getting item...')
        return self._cities[s]

    def __iter__(self):
        print('Calling Cities instance __iter__')
        return self.CityIterator(self)

    class CityIterator:
        def __init__(self, city_obj):
            # cities is an instance of Cities
            print('Calling CityIterator __init__')
            self._city_obj = city_obj
            self._index = 0

        def __iter__(self):
            print('Calling CitiyIterator instance __iter__')
            return self

        def __next__(self):
            print('Calling __next__')
            if self._index >= len(self._city_obj):
                raise StopIteration
            else:
                item = self._city_obj._cities[self._index]
                self._index += 1
                return item

python

# IO

# file open

# file lines

# file state

# os.walk

# StringIO

# shutil.move and os.rename

# shutil.copy2 copy file with metadata

# print

# print %s, %r

# fstring

# logical lines

# Container

# string index

# string interning

# + and +=

# collections type check

# iterable

# iter

# generator

# generator state

# yield from

# defaultdict realize multidict

# zip and zip_longest

# dict

# dict_keys and dict_items

# dict & set comprehension

# dict update

# dict sort by values

# UserDict

# ChainMap

# MappingProxyType

# deque

# tuple

# tuple compare

# typing namedtuple

# itertools

# itertools count

# itertools cycle

# itertools repeat

# itertools tee

# itertools accumulate

# itertools chain

# itertools compress

# itertools dropwhile

# itertools takenwhile

# itertools filterfalse

# itertools groupby

# itertools islice

# itertools starmap

# itertools zip_longest

# itertools batched

# itertools pairwise

# itertools product

# itertools combinations_with_replacement

# itertools combinations

# itertools permutations

# itertools accumulate

# exmaple

# slice

# hashable

# set membership is much faster than list or tuple membership

# enumerate start

# Counter

# TypedDict

# itemgetter

# remove list duplicates and keep order

# reversed

# flatten list

# secrets password generator

# sorted

# random

# random choices

# random sample

# Class

# super()

# __class__ point to class

# calendar

# type() and isinstance

# typing udt(user-defined types)

# class point to class

# slots

# iter

# format

# sub

# call

# fun code

# not logic in `main`