download all files in specific folder from hugging face model

 refer to code:

.

# Download all files from the IP-Adapter/sdxl_models folder

from huggingface_hub import snapshot_download

# Download the sdxl_models folder and its contents

snapshot_download(

repo_id="h94/IP-Adapter",

repo_type="model",

local_dir="./IP-Adapter_sdxl_models",

allow_patterns=["sdxl_models/*"]

)

..

It download all files under the sdxl_models folder.

Thank you.

Download specific model from hugging face

 refer to code

.

import os

import shutil

from huggingface_hub import hf_hub_download

# Repository name

repo_id = "h94/IP-Adapter"

# Directory to save the downloaded files

local_directory = "./models/image_encoder"

# Ensure the local directory exists

os.makedirs(local_directory, exist_ok=True)

# List of files to download

files_to_download = [

"models/image_encoder/config.json",

"models/image_encoder/model.safetensors",

"models/image_encoder/pytorch_model.bin"

]

# Download each file and move it to the desired directory

for file in files_to_download:

file_path = hf_hub_download(repo_id=repo_id, filename=file)

# Construct the destination path

dest_path = os.path.join(local_directory, os.path.basename(file))

# Move the file to the destination path

shutil.move(file_path, dest_path)

print(f"Downloaded and moved to {dest_path}")

..

Thank you.

other option is

file_path = hf_hub_download(repo_id=repo_id, filename=file, cache_dir=local_directory, force_download=True)

draw image as grid on notebook

simply check below code and example result.

.

def image_grid(imgs, rows, cols):

assert len(imgs) == rows*cols

w, h = imgs[0].size

grid = Image.new('RGB', size=(cols*w, rows*h))

grid_w, grid_h = grid.size

for i, img in enumerate(imgs):

grid.paste(img, box=(i%cols*w, i//cols*h))

return grid

# read image prompt

image = Image.open("assets/images/statue.png")

depth_map = Image.open("assets/structure_controls/depth.png")

image_grid([image.resize((256, 256)), depth_map.resize((256, 256))], 1, 2)

..

Thank you.

Unknown parameter in retrievalConfiguration.vectorSearchConfiguration: "overrideSearchType", must be one of: numberOfResults

 Error in AWS bedrock like:

Unknown parameter in retrievalConfiguration.vectorSearchConfiguration: "overrideSearchType", must be one of: numberOfResults

Solution 

Update boto3 sdk as latest one.

The parameters changed on 2024-03-27.

refer to here: https://awsapichanges.com/archive/changes/cd42c1-bedrock-agent-runtime.html

Thank you!

Stop window update process in force

Open powershell by admin 

and put this on terminal

> net stop wuauserv #stop service update
> net stop bits #stop bits
> Remove-Item -Path "C:\Windows\SoftwareDistribution\Download\*" -Recurse -Force
> net start wuauserv #restart 
> net start bits #restart

You can do something else after stop bits, service update.

Thank you.

Embedding invisible water mark on image

 Firstly, install opencv

pip install opencv-python numpy

code for invisible water mark embedding 

import cv2

import numpy as np

def embed_watermark(image_path, watermark, output_path):

# Load the image

img = cv2.imread(image_path)

# Ensure the image is in 3 channels RGB

if img.shape[2] != 3:

print("Image needs to be RGB")

return

# Prepare the watermark

# For simplicity, the watermark is repeated to match the image size

watermark = (watermark * (img.size // len(watermark) + 1))[:img.size]

watermark = np.array(list(watermark), dtype=np.uint8).reshape(img.shape)

# Embed watermark by altering the least significant bit

img_encoded = img & ~1 | (watermark & 1)

# Save the watermarked image

cv2.imwrite(output_path, img_encoded)

print("Watermarked image saved to", output_path)

# Usage

embed_watermark('path_to_your_image.jpg', 'your_watermark_text', 'watermarked_image.jpg')

retrive code

def extract_watermark(watermarked_image_path, original_image_path, output_path):

# Load the watermarked and the original image

img_encoded = cv2.imread(watermarked_image_path)

img_original = cv2.imread(original_image_path)

# Extract the watermark by comparing the least significant bits

watermark = img_encoded & 1 ^ img_original & 1

watermark = (watermark * 255).astype(np.uint8) # Scale to 0-255 for visibility

# Save or display the extracted watermark

cv2.imwrite(output_path, watermark)

print("Extracted watermark saved to", output_path)

# Usage

extract_watermark('watermarked_image.jpg', 'path_to_your_image.jpg', 'extracted_watermark.jpg')

Error: "No data found. 0 train images with repeating" in LoRA Training with kohya-ss/sd-scripts.

 This is my cmd for training lora with kohya-ss/sd-scripts

python -m accelerate.commands.launch --num_cpu_threads_per_process=8 \

...

--enable_bucket \

...

--train_data_dir="/images/" \

--output_dir="models/loras" \

--logging_dir="./logs" \

--log_prefix=silostyle \

--resolution=512,512 \

--network_module=networks.lora \

...

And I met this error

> No data found.

   ...

   0 train images with repeating

   ..

One thing to check is folder name.

Your images and caption txt file has sub folder and the folder name should have "number_title"

correct :  /images/1_files, /images/2_files

wrong : /images/files, /images/0_files 

I am figuring out why kohya has such a rule..

Thank you.

             

Image Tagging workflow for comfyUI

 

Image Tagging workflow for comfyUI

More details:

https://www.marearts.com/marearts/WD14-Image-Tagging-for-ComfyUI-Workflow-e00b63d92ce9434aaa22c554521de614

Thank you!

Kill process which use specific port

1. find pid

> lsof -i :8188

COMMAND PID USER FD TYPE DEVICE SIZE/OFF NODE NAME

Code\x20H 76433 ccxxxxx 46u IPv4 0xc8a9187a02c84768 0t0 TCP localhost:8188 (LISTEN)

2. kill process

> kill -9 76433

That's it
Thank you!

pip-tools or poetry which offer more robust dependency resolution mechanisms than pip's default resolver.

 

To help with resolving complex dependency graphs, you can use tools like pip-tools or poetry which offer more robust dependency resolution mechanisms than pip's default resolver.

Pip-tools:
pip install pip-tools
pip-compile --output-file requirements.txt
pip-sync requirements.txt

Poetry:
pip install poetry
poetry init
poetry add <package-name>

Thank you

🙇🏻‍♂️

git tip, delete all history (commit) information

 

.

# Warning: This will destroy your current history.

# Ensure that you understand the consequences before running these commands.

# Go to your repository directory

cd path/to/your/repository

# Create a new initial commit

git checkout --orphan newBranch

git add -A

git commit -m "Initial commit"

# Delete the main branch

git branch -D main

# Rename the current branch to main

git branch -m main

# Force push to overwrite the history on the remote repository

git push --force origin main

..

install dlib on ubuntu

 

1.

sudo apt update

sudo apt install cmake

2.

pip install cmake

3.

pip install dlib

Check amout of size with consider .gitignore files.

Put this cmd. 

git ls-files \

  | xargs du -ch \

  | grep total$

Thank you!

comfyui custom node developing v2

 add one more class 

save image to png 

.

# from .imagetransfer import ImageTransfer

# __all__ = ['NODE_CLASS_MAPPINGS', 'NODE_DISPLAY_NAME_MAPPINGS']

from PIL import Image

import torch

from torchvision.transforms.functional import to_pil_image, to_tensor

import os

import folder_paths

from comfy.cli_args import args

from PIL.PngImagePlugin import PngInfo

import json

import numpy as np

class ImageTransfer:

@classmethod

def INPUT_TYPES(cls):

return {

"required": {

"image": ("IMAGE",),

},

}

RETURN_TYPES = ("IMAGE",)

FUNCTION = "transfer"

CATEGORY = "Utility"

def transfer(self, image):

# Ensure we're not tracking gradients for this operation

print(f"Type of the image tensor: {image.dtype}")

print(f"Dimensions of the image tensor: {image.shape}")

with torch.no_grad():

# Check if image tensor is in the expected format [1, H, W, C]

if image.dim() == 4 and image.shape[-1] in [1, 3]:

# Permute the tensor to match [C, H, W] format expected by to_pil_image

image_permuted = image.squeeze(0).permute(2, 0, 1)

# Convert the PyTorch tensor to a PIL Image

image_pil = to_pil_image(image_permuted)

# Resize the PIL Image

resized_image_pil = image_pil.resize((256, 256), Image.LANCZOS)

# Convert the PIL Image back to a PyTorch tensor

resized_image_tensor = to_tensor(resized_image_pil)

# Permute dimensions back to [1, H, W, C] and add the batch dimension

resized_image_tensor = resized_image_tensor.permute(1, 2, 0).unsqueeze(0)

print(f"Type of the resized image tensor: {resized_image_tensor.dtype}")

print(f"Dimensions of the resized image tensor: {resized_image_tensor.shape}")

return (resized_image_tensor,)

else:

raise ValueError("Image tensor format not recognized. Expected format: [1, H, W, C].")

class Save_png2svg:

def __init__(self):

self.output_dir = folder_paths.get_output_directory()

self.type = "output"

self.prefix_append = ""

self.compress_level = 4

@classmethod

def INPUT_TYPES(s):

return {"required":

{"images": ("IMAGE", ),

"filename_prefix": ("STRING", {"default": "ComfyUI"})},

"hidden": {"prompt": "PROMPT", "extra_pnginfo": "EXTRA_PNGINFO"},

}

RETURN_TYPES = ()

FUNCTION = "Save_png2svg"

OUTPUT_NODE = True

CATEGORY = "image"

def Save_png2svg(self, images, filename_prefix="ComfyUI", prompt=None, extra_pnginfo=None):

filename_prefix += self.prefix_append

full_output_folder, filename, counter, subfolder, filename_prefix = folder_paths.get_save_image_path(filename_prefix, self.output_dir, images[0].shape[1], images[0].shape[0])

results = list()

for (batch_number, image) in enumerate(images):

i = 255. * image.cpu().numpy()

img = Image.fromarray(np.clip(i, 0, 255).astype(np.uint8))

metadata = None

if not args.disable_metadata:

metadata = PngInfo()

if prompt is not None:

metadata.add_text("prompt", json.dumps(prompt))

if extra_pnginfo is not None:

for x in extra_pnginfo:

metadata.add_text(x, json.dumps(extra_pnginfo[x]))

filename_with_batch_num = filename.replace("%batch_num%", str(batch_number))

file = f"{filename_with_batch_num}_{counter:05}_.png"

img.save(os.path.join(full_output_folder, file), pnginfo=metadata, compress_level=self.compress_level)

results.append({

"filename": file,

"subfolder": subfolder,

"type": self.type

})

counter += 1

return { "ui": { "images": results } }

NODE_CLASS_MAPPINGS = {

"ImageTransfer": ImageTransfer,

"Save_png2svg":Save_png2svg

}

NODE_DISPLAY_NAME_MAPPINGS = {

"ImageTransfer": "Image Transfer",

"Save_png2svg": "Save png2svg"

}

..

comfyui image resize simple custom node code

 refer to code for your scratch 

:

# from .imagetransfer import ImageTransfer

# __all__ = ['NODE_CLASS_MAPPINGS', 'NODE_DISPLAY_NAME_MAPPINGS']

from PIL import Image

import torch

from torchvision.transforms.functional import to_pil_image, to_tensor

class ImageTransfer:

@classmethod

def INPUT_TYPES(cls):

return {

"required": {

"image": ("IMAGE",),

},

}

RETURN_TYPES = ("IMAGE",)

FUNCTION = "transfer"

CATEGORY = "Utility"

def transfer(self, image):

# Ensure we're not tracking gradients for this operation

print(f"Type of the image tensor: {image.dtype}")

print(f"Dimensions of the image tensor: {image.shape}")

with torch.no_grad():

# Check if image tensor is in the expected format [1, H, W, C]

if image.dim() == 4 and image.shape[-1] in [1, 3]:

# Permute the tensor to match [C, H, W] format expected by to_pil_image

image_permuted = image.squeeze(0).permute(2, 0, 1)

# Convert the PyTorch tensor to a PIL Image

image_pil = to_pil_image(image_permuted)

# Resize the PIL Image

resized_image_pil = image_pil.resize((256, 256), Image.LANCZOS)

# Convert the PIL Image back to a PyTorch tensor

resized_image_tensor = to_tensor(resized_image_pil)

# Permute dimensions back to [1, H, W, C] and add the batch dimension

resized_image_tensor = resized_image_tensor.permute(1, 2, 0).unsqueeze(0)

print(f"Type of the resized image tensor: {resized_image_tensor.dtype}")

print(f"Dimensions of the resized image tensor: {resized_image_tensor.shape}")

return (resized_image_tensor,)

else:

raise ValueError("Image tensor format not recognized. Expected format: [1, H, W, C].")

NODE_CLASS_MAPPINGS = {

"ImageTransfer": ImageTransfer

}

NODE_DISPLAY_NAME_MAPPINGS = {

"ImageTransfer": "Image Transfer"

}

Run multi comfyUI

 make different port number

Window

>.\python_embeded\python.exe -s ComfyUI\main.py --port 8189 --windows-standalone-build

Linux or Mac

> python ./main.py --port 8189 --windows-standalone-build

Thank you.

🙇🏻‍♂️

Search process by port number on ubuntu and kill it.

 

>sudo lsof -i :8188

python  231134 mare   45u  IPv4 2074469      0t0  TCP localhost:8188->localhost:42132 (CLOSE_WAIT)

python  231134 mare   46u  IPv4 2105633      0t0  TCP localhost:8188->localhost:57870 (CLOSE_WAIT)

python  231134 mare   47u  IPv4 2074473      0t0  TCP localhost:8188->localhost:42160 (CLOSE_WAIT)

python  231134 mare   48u  IPv4 2103693      0t0  TCP localhost:8188->localhost:57886 (CLOSE_WAIT)

> kill -9 231134

🙇🏻‍♂️

Retrieve my ssh keys and generate

 

Navigate to your SSH directory

cd ~/.ssh

List the RSA public key files:

ls -l *.pub

View the Contents of Your RSA Public Key

cat id_rsa.pub

To generate a new SSH RSA key pair, you can use the following command

ssh-keygen -t rsa -b 4096

Thank you. marearts.com

🙇🏻‍♂️

Create New swap file

 

Create a New Swap File

sudo fallocate -l 2G /swapfile2

sudo chmod 600 /swapfile2

sudo mkswap /swapfile2

sudo swapon /swapfile2

Make the swap file permanent

echo '/swapfile2 none swap sw 0 0' | sudo tee -a /etc/fstab

Check Current swap status

sudo swapon --show

Thank you 

🙇🏻‍♂️

How to stop docker under Linux

 I can stop docker after put two commend. 

> sudo systemctl stop docker

[sudo] password for mare: 

Warning: Stopping docker.service, but it can still be activated by:

  docker.socket

> sudo systemctl stop docker.socket

I hope it's helpful to you.

Thank you.

🙇🏻‍♂️

Dominant frequency extraction.

 

Let's say we have channel x Length signal data ex)EEG (electroencephalogram) or time series data.

We might wonder what dominant Hz is there.

The code analysis this question and return 5 top dominant frequency. 

.

import numpy as np

from collections import Counter

from scipy.signal import welch

def identify_dominant_frequencies(signal, fs, top_n=5):

freqs, psd = welch(signal, fs)

peak_indices = np.argsort(psd)[-top_n:]

dominant_freqs = freqs[peak_indices]

return dominant_freqs

..

dominant_freqs = identify_dominant_frequencies(signal, fs, top_n)

dominant_freqs_summary[channel].extend(dominant_freqs) # Append the frequencies

..

median_dominant_freqs = {channel: np.median(freqs) if freqs else None for channel, freqs in dominant_freqs_summary.items()}

..

def get_top_n_frequencies(freq_list, top_n=5, bin_width=1.0):

# Bin frequencies into discrete intervals

binned_freqs = np.round(np.array(freq_list) / bin_width) * bin_width

# Count the frequency of each binned frequency

freq_counter = Counter(binned_freqs)

# Find the top N most common binned frequencies

top_freqs = freq_counter.most_common(top_n)

# Extract just the frequencies from the top N tuples (freq, count)

top_freqs = [freq for freq, count in top_freqs]

return top_freqs

# Initialize a dictionary to store the top 5 frequencies for each channel

top_5_freqs_all_channels = {}

bin_width = 1.0

# Calculate the top 5 frequencies for each channel

for channel, freqs in dominant_freqs_summary.items():

top_5_freqs = get_top_n_frequencies(freqs, top_n=5, bin_width=bin_width)

top_5_freqs_all_channels[channel] = top_5_freqs

print(f"{channel}: Top 5 Frequencies = {top_5_freqs}")

..

GroupShuffleSplit, sklearn

 

There are same eeg_id in data, but we can split it based on same id to train, val using GroupShuffleSplit.

Refer to code:

.

import pandas as pd

from sklearn.model_selection import GroupShuffleSplit

# Load your dataset

train = pd.read_csv('./train.csv')

# Display the shape of the dataset

print("Dataset shape:", train.shape)

# Count unique eeg_id values

unique_eeg_id_count = train['eeg_id'].nunique()

print("Unique eeg_id count:", unique_eeg_id_count)

# Initialize the GroupShuffleSplit

gss = GroupShuffleSplit(n_splits=1, test_size=0.2, random_state=42)

# Split the dataset based on the 'eeg_id' to ensure group cohesion

for train_idx, val_idx in gss.split(train, groups=train['eeg_id']):

train_set = train.iloc[train_idx]

val_set = train.iloc[val_idx]

# Now, train_set and val_set are split according to unique eeg_ids,

# ensuring that all records of a single eeg_id are in the same subset

print("Training set shape:", train_set.shape)

print("Validation set shape:", val_set.shape)

..

Thank you.

🙇🏻‍♂️