Do It Again

// ---=+=--- ---=+=--- ---=+=--- ---=+=--- ---=+=--- ---=+=--- ---=+=---

// THE JAMES DEAN PROGRAM!

// VERSION: 002.06 (.REVISION, OR, 206 REVISIONS TOTAL)

// TODO LIST

// 1) ADD REAL ARBITRARY COMPONENT DEVICES (LOOK AT THE PREVIOUS JAMES DEAN)
// 2) ADD CHRONO TIMING FOR MATRIX CREATION (*DONE)
// 3) ADD GLOBAL CHRONO TIMING (*DONE)
// 4) ADD OUTPUT (STD::COUT) FOR THE INDIVIDUAL MATRICES

// ---=+=--- ---=+=--- ---=+=--- ---=+=--- ---=+=--- ---=+=--- ---=+=---
// RECORD OF FATE

// 1930019299 people received data from 002.06-A

// ---=+=--- ---=+=--- ---=+=--- ---=+=--- ---=+=--- ---=+=--- ---=+=---
// STANDARD INCLUDES

#include <iostream>
#include <vector>
#include <string>
#include <bitset>
#include <fstream>

// ---=+=--- ---=+=--- ---=+=--- ---=+=--- ---=+=--- ---=+=--- ---=+=---
// THREAD INCLUDES

#include <thread>

// ---=+=--- ---=+=--- ---=+=--- ---=+=--- ---=+=--- ---=+=--- ---=+=---
// TIME INCLUDES

#include <chrono>

// ---=+=--- ---=+=--- ---=+=--- ---=+=--- ---=+=--- ---=+=--- ---=+=---
// LOCKING

#include <mutex>

// ---=+=--- ---=+=--- ---=+=--- ---=+=--- ---=+=--- ---=+=--- ---=+=---
// RANDOM INCLUDES

#include <random>
#include "random/pcg_random.hpp"
#include "random/pcg_extras.hpp"
#include "random/pcg_uint128.hpp"

// -------------------------------====++====-----------------------------------
// MULTIPRECISION INCLUDES

 #include <boost/multiprecision/cpp_int.hpp>
 #include <boost/multiprecision/cpp_dec_float.hpp> 
 
// -------------------------------====++====-----------------------------------
// USING DECLARATIVES

using boost::multiprecision::cpp_int;
using mp_float = boost::multiprecision::number<boost::multiprecision::cpp_dec_float<0>>;

// ---=+=--- ---=+=--- ---=+=--- ---=+=--- ---=+=--- ---=+=--- ---=+=---
// CLASSES

// ---=+=--- ---=+=--- ---=+=--- ---=+=--- ---=+=--- ---=+=--- ---=+=---
// FORWARD DECLARED CLASSES

class positive_hex;
class data_buff;
class hash_table;
class arbitrary_component_device;
class technical;
class level_table;

// ---=+=--- ---=+=--- ---=+=--- ---=+=--- ---=+=--- ---=+=--- ---=+=---
// JAMES DEAN DATA

class james_dean_data
{
	protected:
	
		cpp_int local_system_id = 0;
		
		cpp_int local_production_id = 0;
		cpp_int local_data_id = 0;
		cpp_int local_class_id = 0;
		cpp_int local_type_id = 0;
		
		std::vector<std::vector<cpp_int>> local_random_matrix;
		std::vector<std::vector<cpp_int>> local_mod_random_matrix;
		
		std::vector<std::vector<positive_hex*>> local_positive_hex_matrix;
		std::vector<std::vector<data_buff*>> local_data_buff_matrix;
		std::vector<std::vector<arbitrary_component_device*>> arbitrary_component_device_matrix;
		std::vector<std::vector<hash_table*>> local_hash_table_matrix;
		std::vector<std::vector<technical*>> local_technical_matrix;
		
		std::vector<level_table*> level_table_matrix;
		
		void set_level_table(std::vector<level_table*> level_table_input)
		{
			level_table_matrix = level_table_input;
		}
		
		void set_technical_matrix(std::vector<std::vector<technical*>> technical_input)
		{
			local_technical_matrix = technical_input;
		}
		
		void set_arbitrary_component_device_matrix(std::vector<std::vector<arbitrary_component_device*>> arbitrary_component_device_input)
		{
			arbitrary_component_device_matrix = arbitrary_component_device_input;
		}
		
		void set_local_positive_hex_matrix(std::vector<std::vector<positive_hex*>> positive_hex_input)
		{
			local_positive_hex_matrix = positive_hex_input;
		}
		
		void set_local_data_buff_matrix(std::vector<std::vector<data_buff*>> data_buff_input)
		{
			local_data_buff_matrix = data_buff_input;
		}
		
		void set_local_random_matrix(std::vector<std::vector<cpp_int>> random_matrix_input)
		{
			local_random_matrix = random_matrix_input;
		}
		
		void set_local_mod_random_matrix(std::vector<std::vector<cpp_int>> mod_random_matrix_input)
		{
			local_mod_random_matrix = mod_random_matrix_input;
		}
		
		void set_local_system_id(cpp_int system_id_input)
		{
			local_system_id = system_id_input;
		}
		
		void set_local_production_id(cpp_int production_id_input)
		{
			local_production_id = production_id_input;
		}
		
		void set_local_data_id(cpp_int data_id_input)
		{
			local_data_id = data_id_input;
		}
		
		void set_local_class_id(cpp_int class_id_input)
		{
			local_class_id = class_id_input;
		}
		
		void set_local_type_id(cpp_int type_id_input)
		{
			local_type_id = type_id_input;
		}
		
		void set_local_hash_table_matrix(std::vector<std::vector<hash_table*>> hash_table_input)
		{
			local_hash_table_matrix = hash_table_input;
		}
		
	public:
	
		void call_set_level_table(std::vector<level_table*> call_level_table_input)
		{
			set_level_table(call_level_table_input);
		}
	
		void call_set_technical_matrix(std::vector<std::vector<technical*>> call_technical_input)
		{
			set_technical_matrix(call_technical_input);
		}
	
		void call_set_local_hash_table_matrix(std::vector<std::vector<hash_table*>> call_hash_table_input)
		{
			set_local_hash_table_matrix(call_hash_table_input);
		}
	
		void call_set_arbitrary_component_device_matrix(std::vector<std::vector<arbitrary_component_device*>> call_arbitrary_component_device_input)
		{
			set_arbitrary_component_device_matrix(call_arbitrary_component_device_input);
		}
		
		void call_set_local_positive_hex_matrix(std::vector<std::vector<positive_hex*>> call_positive_hex_input)
		{
			set_local_positive_hex_matrix(call_positive_hex_input);
		}
		
		void call_set_local_data_buff_matrix(std::vector<std::vector<data_buff*>> call_data_buff_input)
		{
			set_local_data_buff_matrix(call_data_buff_input);
		}
		
		void call_set_local_random_matrix(std::vector<std::vector<cpp_int>> call_random_matrix_input)
		{
			set_local_random_matrix(call_random_matrix_input);
		}
		
		void call_set_local_mod_random_matrix(std::vector<std::vector<cpp_int>> call_mod_random_matrix_input)
		{
			set_local_mod_random_matrix(call_mod_random_matrix_input);
		}
	
		void call_set_local_system_id(cpp_int call_system_id_input)
		{
			set_local_system_id(call_system_id_input);
		}
		
		void call_set_local_production_id(cpp_int call_production_id_input)
		{
			set_local_production_id(call_production_id_input);
		}
		
		void call_set_local_data_id(cpp_int call_data_id_input)
		{
			set_local_data_id(call_data_id_input);
		}
		
		void call_set_local_class_id(cpp_int call_class_id_input)
		{
			set_local_class_id(call_class_id_input);
		}
		
		void call_set_local_type_id(cpp_int call_type_id_input)
		{
			set_local_type_id(call_type_id_input);
		}
};

// ---=+=--- ---=+=--- ---=+=--- ---=+=--- ---=+=--- ---=+=--- ---=+=---
// LEVEL TABLE

class level_table : public james_dean_data
{
	protected:
		
		std::vector<std::vector<cpp_int>> level_table_matrix;
		std::vector<std::vector<std::vector<cpp_int>>> table_matrices;
		
		void set_table_matrices(std::vector<std::vector<std::vector<cpp_int>>> table_matrices_input)
		{
			table_matrices = table_matrices_input;
		}
		
		void set_level_table_matrix(std::vector<std::vector<cpp_int>> level_table_matrix_input)
		{
			level_table_matrix = level_table_matrix_input;
		}
		
	public:
	
		void call_set_table_matrices(std::vector<std::vector<std::vector<cpp_int>>> call_table_matrices_input)
		{
			set_table_matrices(call_table_matrices_input);
		}
	
		void call_set_level_table_matrix(std::vector<std::vector<cpp_int>> call_level_table_matrix_input)
		{
			set_level_table_matrix(call_level_table_matrix_input);
		}
};

// ---=+=--- ---=+=--- ---=+=--- ---=+=--- ---=+=--- ---=+=--- ---=+=---
// DATA BUFF

class data_buff : public james_dean_data
{
	protected:
		
	public:
	
};

// ---=+=--- ---=+=--- ---=+=--- ---=+=--- ---=+=--- ---=+=--- ---=+=---
// POSITIVE HEX

class positive_hex : public james_dean_data
{
	protected:

	public:
	
};

// ---=+=--- ---=+=--- ---=+=--- ---=+=--- ---=+=--- ---=+=--- ---=+=---
// HASH TABLE

class hash_table : public james_dean_data
{
	protected:
	
		std::vector<std::vector<std::string*>> local_hash_table_vector;
		
		void set_local_hash_table(std::vector<std::vector<std::string*>> hash_table_input)
		{
			local_hash_table_vector = hash_table_input;
		}

	public:
		
		void call_set_local_hash_table(std::vector<std::vector<std::string*>> call_hash_table_input)
		{
			set_local_hash_table(call_hash_table_input);
		}
};

// ---=+=--- ---=+=--- ---=+=--- ---=+=--- ---=+=--- ---=+=--- ---=+=---
// ARBITRARY COMPONENT DEVICE

class arbitrary_component_device : public james_dean_data
{
	protected:
	
	public:
	
};

// POPULUS

class civilian
{
	protected:
	
		std::vector<james_dean_data*> civilian_james_dean_matrix;
		
		cpp_int local_civilian_id = 0;
		
		void set_civilian_james_dean_matrix(std::vector<james_dean_data*> james_dean_input)
		{
			civilian_james_dean_matrix = james_dean_input;
		}
		
		void set_civilian_id(cpp_int civilian_id_input)
		{
			local_civilian_id = civilian_id_input;
		}
	
	public:
	
		void call_set_civilian_james_dean_matrix(std::vector<james_dean_data*> call_james_dean_input)
		{
			set_civilian_james_dean_matrix(call_james_dean_input);
		}
		
		void call_set_civilian_id(cpp_int call_civilian_id_input)
		{
			set_civilian_id(call_civilian_id_input);
		}
};

// ---=+=--- ---=+=--- ---=+=--- ---=+=--- ---=+=--- ---=+=--- ---=+=---
// TECHNICAL
// JUST A WORD FOR CORNERING, FOLDING, JUMPING.

class technical : public james_dean_data
{
	protected:
	
		std::string name_str = "void";
		signed long long int technical_id = 0;
		
		signed long long int program_version = 0;
		signed long long int program_revision = 0;
		
		void set_program_version(signed long long int input_int)
		{
			program_version = input_int;
		}
		
		void set_program_revision(signed long long int input_int)
		{
			program_revision = input_int;
		}
		
		void set_name_str(std::string input_str)
		{
			name_str = "technical_" + std::to_string(technical_id) + "_of_" + std::to_string(program_version) + "." + std::to_string(program_revision);
		}
		
		void display_name_str()
		{
			std::cout << name_str << "\n";
		}
		
	public:
	
		void call_set_program_version(signed long long int call_input_int)
		{
			set_program_version(call_input_int);
		}
		
		void call_set_program_revision(signed long long int call_input_int)
		{
			set_program_revision(call_input_int);
		}
		
		void call_set_name_str(std::string call_input_str)
		{
			set_name_str(call_input_str);
		}
		
		void call_display_name_str()
		{
			display_name_str();
		}
	
};

// ---=+=--- ---=+=--- ---=+=--- ---=+=--- ---=+=--- ---=+=--- ---=+=---
// FORWARD DECLARATIONS

std::vector<std::vector<cpp_int>> return_random_matrix(cpp_int);
std::vector<std::vector<data_buff*>> return_data_buff_matrix(cpp_int);
std::vector<std::vector<positive_hex*>> return_positive_hex_matrix(cpp_int);
std::vector<std::vector<arbitrary_component_device*>> return_arbitrary_component_device_matrix(cpp_int);
std::vector<std::vector<hash_table*>> return_hash_table_matrix(cpp_int);
std::vector<std::vector<technical*>> return_technical_matrix(cpp_int);

void create_and_populate(cpp_int);

std::vector<level_table*> return_level_table(cpp_int);


// ---=+=--- ---=+=--- ---=+=--- ---=+=--- ---=+=--- ---=+=--- ---=+=---
// STRUCTURES

struct main_data_struct {

	cpp_int local_civilian_id = 0;
	std::mutex civilian_id_mutex;
	
	std::mutex last_run_mutex;
	bool last_run = false;
	
	cpp_int local_civil_reservation_number = 500'000'000;
	cpp_int world_population = 8'231'613'070; // June 30th, 2025 (on May 12th, 2025)
	
	cpp_int local_system_id = 0;
	std::mutex system_id_mutex;
	
	cpp_int local_production_id = 0;
	std::mutex production_id_mutex;
	
	cpp_int local_data_id = 0;
	std::mutex data_id_mutex;
	
	cpp_int local_class_id = 0;
	std::mutex class_id_mutex;
	
	cpp_int local_type_id = 0;
	std::mutex type_id_mutex;
	
	cpp_int local_random_production_id = 0;
	std::mutex random_production_id_mutex;
	
	cpp_int local_random_data_id = 0;
	std::mutex random_data_id_mutex;
	
	cpp_int local_random_class_id = 0;
	std::mutex random_class_id_mutex;
	
	cpp_int local_random_type_id = 0;
	std::mutex random_type_id_mutex;
	
	cpp_int local_mod_random_production_id = 0;
	std::mutex mod_random_production_id_mutex;
	
	cpp_int local_mod_random_data_id = 0;
	std::mutex mod_random_data_id_mutex;
	
	cpp_int local_mod_random_class_id = 0;
	std::mutex mod_random_class_id_mutex;
	
	cpp_int local_mod_random_type_id = 0;
	std::mutex mod_random_type_id_mutex;
	
	cpp_int local_data_buff_production_id = 0;
	std::mutex data_buff_production_id_mutex;
	
	cpp_int local_data_buff_data_id = 0;
	std::mutex data_buff_data_id_mutex;
	
	cpp_int local_data_buff_class_id = 0;
	std::mutex data_buff_class_id_mutex;
	
	cpp_int local_data_buff_type_id = 0;
	std::mutex data_buff_type_id_mutex;
	
	cpp_int local_positive_hex_production_id = 0;
	std::mutex positive_hex_production_id_mutex;
	
	cpp_int local_positive_hex_data_id = 0;
	std::mutex positive_hex_data_id_mutex;
	
	cpp_int local_positive_hex_class_id = 0;
	std::mutex positive_hex_class_id_mutex;
	
	cpp_int local_positive_hex_type_id = 0;
	std::mutex positive_hex_type_id_mutex;
	
	cpp_int local_arbitrary_component_device_production_id = 0;
	std::mutex arbitrary_component_device_production_id_mutex;
	
	cpp_int local_arbitrary_component_device_data_id = 0;
	std::mutex arbitrary_component_device_data_id_mutex;
	
	cpp_int local_arbitrary_component_device_class_id = 0;
	std::mutex arbitrary_component_device_class_id_mutex;
	
	cpp_int local_arbitrary_component_device_type_id = 0;
	std::mutex arbitrary_component_device_type_id_mutex;
	
	cpp_int local_hash_table_production_id = 0;
	std::mutex hash_table_production_id_mutex;
	
	cpp_int local_hash_table_data_id = 0;
	std::mutex hash_table_data_id_mutex;
	
	cpp_int local_hash_table_class_id = 0;
	std::mutex hash_table_class_id_mutex;
	
	cpp_int local_hash_table_type_id = 0;
	std::mutex hash_table_type_id_mutex;
	
	cpp_int production_target = 8;
	cpp_int data_target = 10;
	cpp_int class_target = 15;
	cpp_int type_target = 15;
	
	cpp_int random_matrix_vector_target = 25;
	
	cpp_int mod_random_matrix_vector_target = 25;
	
	
	cpp_int hash_table_matrix_vector_target = 10;
	
	cpp_int data_buff_matrix_vector_target = 10;
	
	cpp_int positive_hex_matrix_vector_target = 10;
		
	cpp_int arbitrary_component_device_matrix_vector_target = 10;
		
	std::vector<james_dean_data*> local_james_dean_vector;
	
	cpp_int total_data_counter = 0;
	cpp_int positive_hex_count = 0;
	cpp_int data_buff_count = 0;
	cpp_int arbitrary_component_device_count = 0;
	
	cpp_int positive_hex_target = 15;
	cpp_int data_buff_target = 15;
	cpp_int hash_table_target = 15;
	cpp_int random_target = 20;
	cpp_int arbitrary_component_device_target = 15;
	
	std::mutex print_mutex;
	
	std::mutex james_dean_vector_mutex;
	std::mutex data_counter_mutex;
	std::mutex technical_print_mutex;
	std::mutex data_buff_count_mutex;
	std::mutex positive_hex_count_mutex;
	std::mutex arbitrary_component_device_count_mutex;
	std::mutex technical_mutex;
	std::mutex arbitrary_component_device_mutex;
	std::mutex hash_table_mutex;
	std::mutex mod_random_mutex;
	std::mutex random_mutex;
	std::mutex positive_hex_mutex;
	std::mutex data_buff_mutex;
	
	std::mutex data_buff_print_mutex;
	
	cpp_int data_buff_counter = 0;
	
	std::mutex level_table_mutex;
	cpp_int level_table_target = 40;
	
	
	cpp_int technical_target = 20;
	
	cpp_int local_reserved_id = 0;
	
	
} james_dean;

// ---=+=--- ---=+=--- ---=+=--- ---=+=--- ---=+=--- ---=+=--- ---=+=---
// INT MAIN

int main(int argc, char *argv[])
{
	signed long long int program_version = 002;
	signed long long int program_revision = 06;
	
	std::cout << "\n\nC++ JAMES DEAN DATA CREATOR\n\nVERSION: " << program_version << " REVISION: " << program_revision << "\n\n";
	
	std::vector<std::thread> thread_vector;
	cpp_int thread_target = 24;
	cpp_int local_thread_id = 0;
	
	bool run_program = true;
	
	while (run_program == true)
	{
		std::cout << "[main] run_program begin\n";
		
		while (thread_vector.size() < thread_target)
		{
			std::cout << "[main] thread_vector.size() = " << thread_vector.size() << "\n";
			std::cout << "[main] creating thread " << local_thread_id << "\n";
			thread_vector.emplace_back(create_and_populate, local_thread_id);
			local_thread_id = local_thread_id + 1;
		}
		
		for(signed long long int current_thread_id = 0; current_thread_id < thread_vector.size(); current_thread_id++)
		{
			thread_vector[current_thread_id].join();
		}
		
		if(james_dean.local_production_id >= james_dean.production_target)
		{
			run_program = false;
			
			std::cout << "[main] run_program now set to false (gracefully)\n";
		}
		
		if(thread_vector.size() >= thread_target)
		{
			std::cout << "[main] while loop sleep for 60 seconds, thread_vector.size() = " << thread_vector.size() << "\n";
			std::this_thread::sleep_for(std::chrono::seconds(60));
		}
	}
	
	std::cout << "\n\n\n********************\n********************\nexecution completed!\n\n";
	return(0);
}

// ---=+=--- ---=+=--- ---=+=--- ---=+=--- ---=+=--- ---=+=--- ---=+=---
// FUNCTIONS
		
// ---=+=--- ---=+=--- ---=+=--- ---=+=--- ---=+=--- ---=+=--- ---=+=---
// CREATE AND POPULATE

void create_and_populate(cpp_int thread_id)
{
	cpp_int cpp_int_1 = 1;
	
	std::cout << "[" << thread_id << "] thread " << thread_id << " begin creation of james_dean_data\n";
	
	cpp_int local_data_count = james_dean.local_production_id * james_dean.local_data_id * james_dean.local_class_id * james_dean.local_type_id;
	
	std::vector<std::vector<cpp_int>> local_random_matrix;
	std::vector<cpp_int> small_random_vector;
	
	std::vector<std::vector<cpp_int>> local_mod_random_matrix;
	std::vector<cpp_int> small_mod_random_vector;
	
	std::vector<std::vector<hash_table*>> local_hash_table_matrix;
	std::vector<hash_table*> small_hash_table_vector;
	
	std::vector<std::vector<data_buff*>> local_data_buff_matrix;
	std::vector<data_buff*> small_data_buff_vector;
	
	std::vector<std::vector<positive_hex*>> local_positive_hex_matrix;
	std::vector<positive_hex*> small_positive_hex_vector;
	
	std::vector<std::vector<arbitrary_component_device*>> local_arbitrary_component_device_matrix;
	std::vector<arbitrary_component_device*> small_arbitrary_component_device_vector;
	
	std::vector<std::vector<technical*>> local_technical_matrix;
	std::vector<technical*> small_technical_vector;
	
	std::vector<level_table*> local_level_table_vector;
	
	std::vector<james_dean_data*> local_james_dean_vector;
	
	bool run_program = true;
	
	while (run_program == true)
	{
		std::cout << "[" << thread_id << "] creating james_dean_data object #" << james_dean.local_production_id * james_dean.local_data_id * james_dean.local_class_id * james_dean.local_type_id << " of " << james_dean.production_target * james_dean.data_target * james_dean.class_target * james_dean.type_target << "\n";
		
		std::cout << "[" << thread_id << "] main while loop: " << james_dean.local_production_id << "/" << james_dean.production_target << "\n";
		
		james_dean_data local_james_dean_data;
		
		if(thread_id == 1)
		{
			std::cout << "[" << thread_id << "] setting id\'s for " << james_dean.local_system_id << "::";
			std::cout << james_dean.local_production_id << "/" << james_dean.production_target << "::";
			std::cout <<  james_dean.local_data_id << "/" << james_dean.data_target << "::";
			std::cout << james_dean.local_class_id << "/" << james_dean.class_target << "::";
			std::cout << james_dean.local_type_id << "/" << james_dean.type_target << "\n";
		}
		
		local_james_dean_data.call_set_local_system_id(james_dean.local_system_id);
		james_dean.local_system_id = james_dean.local_system_id + 1;
		
		local_james_dean_data.call_set_local_production_id(james_dean.local_production_id);
		
		local_james_dean_data.call_set_local_data_id(james_dean.local_data_id);
		
		local_james_dean_data.call_set_local_class_id(james_dean.local_class_id);
		
		local_james_dean_data.call_set_local_type_id(james_dean.local_type_id);
		
		auto matrice_start_time = std::chrono::high_resolution_clock::now();
		if(thread_id == 1)
		{
			std::cout << "[" << thread_id << "] creating matrices data_buff, positive_hex, random, mod_random, \n";
			std::cout << "[" << thread_id << "] hash_table, arbitrary_component_device & technical matrices\n";
		}
		
		auto data_buff_start_time = std::chrono::high_resolution_clock::now();
		if(thread_id == 1)
		{
			std::cout << "[" << thread_id << "] creating data_buff matrix\n";
		}
		
		local_data_buff_matrix = return_data_buff_matrix(thread_id);
		local_james_dean_data.call_set_local_data_buff_matrix(local_data_buff_matrix);
		local_data_buff_matrix.clear();
		
		auto data_buff_end_time = std::chrono::high_resolution_clock::now();
		auto data_buff_elapsed_time = data_buff_end_time - data_buff_start_time;
		auto data_buff_nanoseconds = std::chrono::duration_cast<std::chrono::nanoseconds>(data_buff_elapsed_time);
		cpp_int data_buff_nanoseconds_printable = data_buff_nanoseconds.count();
		mp_float data_buff_float_50 = data_buff_nanoseconds_printable.convert_to<mp_float>();
		mp_float data_buff_seconds = data_buff_float_50 / 1'000'000'000;
		std::cout << "[" << thread_id << "] data_buff (" << james_dean.data_buff_target << ") creation time: " << data_buff_seconds << "s\n";
		
		auto positive_hex_start_time = std::chrono::high_resolution_clock::now();
		if(thread_id == 1)
		{
			std::cout << "[" << thread_id << "] creating positive_hex matrix\n";
		}
		
		local_positive_hex_matrix = return_positive_hex_matrix(thread_id);
		local_james_dean_data.call_set_local_positive_hex_matrix(local_positive_hex_matrix);
		local_positive_hex_matrix.clear();
		
		auto positive_hex_end_time = std::chrono::high_resolution_clock::now();
		auto positive_hex_elapsed_time = positive_hex_end_time - positive_hex_start_time;
		auto positive_hex_nanoseconds = std::chrono::duration_cast<std::chrono::nanoseconds>(positive_hex_elapsed_time);
		cpp_int positive_hex_nanoseconds_printable = positive_hex_nanoseconds.count();
		mp_float positive_hex_float_50 = positive_hex_nanoseconds_printable.convert_to<mp_float>();
		mp_float positive_hex_seconds = positive_hex_float_50 / 1'000'000'000;
		std::cout << "[" << thread_id << "] positive_hex (" << james_dean.positive_hex_target << ") creation time: " << positive_hex_seconds << "s\n";
		
		auto random_start_time = std::chrono::high_resolution_clock::now();
		if(thread_id == 1)
		{
			std::cout << "[" << thread_id << "] creating random matrix\n";
		}
		
		local_random_matrix = return_random_matrix(thread_id);
		local_james_dean_data.call_set_local_random_matrix(local_random_matrix);
		local_random_matrix.clear();
		
		auto random_end_time = std::chrono::high_resolution_clock::now();
		auto random_elapsed_time = random_end_time - random_start_time;
		auto random_nanoseconds = std::chrono::duration_cast<std::chrono::nanoseconds>(random_elapsed_time);
		cpp_int random_nanoseconds_printable = random_nanoseconds.count();
		mp_float random_float_50 = random_nanoseconds_printable.convert_to<mp_float>();
		mp_float random_seconds = random_float_50 / 1'000'000'000;
		std::cout << "[" << thread_id << "] random (" << james_dean.positive_hex_target << ") creation time: " << random_seconds << "s\n";
		
		auto mod_random_start_time = std::chrono::high_resolution_clock::now();
		if(thread_id == 1)
		{
			std::cout << "[" << thread_id << "] creating mod_random matrix\n";
		}
		
		local_mod_random_matrix = return_random_matrix(thread_id);
		local_james_dean_data.call_set_local_mod_random_matrix(local_mod_random_matrix);
		local_mod_random_matrix.clear();
		
		auto mod_random_end_time = std::chrono::high_resolution_clock::now();
		auto mod_random_elapsed_time = mod_random_end_time - mod_random_start_time;
		auto mod_random_nanoseconds = std::chrono::duration_cast<std::chrono::nanoseconds>(mod_random_elapsed_time);
		cpp_int mod_random_nanoseconds_printable = mod_random_nanoseconds.count();
		mp_float mod_random_float_50 = mod_random_nanoseconds_printable.convert_to<mp_float>();
		mp_float mod_random_seconds = mod_random_float_50 / 1'000'000'000;
		std::cout << "[" << thread_id << "] mod_random (" << james_dean.random_target << ") creation time: " << mod_random_seconds << "s\n";
		
		auto hash_table_start_time = std::chrono::high_resolution_clock::now();
		if(thread_id == 1)
		{
			std::cout << "[" << thread_id << "] creating hash_table matrix\n";
		}
		
		local_hash_table_matrix = return_hash_table_matrix(thread_id);
		local_james_dean_data.call_set_local_hash_table_matrix(local_hash_table_matrix);
		local_hash_table_matrix.clear();
		
		auto hash_table_end_time = std::chrono::high_resolution_clock::now();
		auto hash_table_elapsed_time = hash_table_end_time - hash_table_start_time;
		auto hash_table_nanoseconds = std::chrono::duration_cast<std::chrono::nanoseconds>(hash_table_elapsed_time);
		cpp_int hash_table_nanoseconds_printable = hash_table_nanoseconds.count();
		mp_float hash_table_float_50 = hash_table_nanoseconds_printable.convert_to<mp_float>();
		mp_float hash_table_seconds = hash_table_float_50 / 1'000'000'000;
		std::cout << "[" << thread_id << "] hash_table (" << james_dean.hash_table_target << ") creation time: " << hash_table_seconds << "s\n";
		
		auto arbitrary_component_device_start_time = std::chrono::high_resolution_clock::now();
		if(thread_id == 1)
		{
			std::cout << "[" << thread_id << "] creating arbitrary_component_device matrix\n";
		}
		
		local_arbitrary_component_device_matrix = return_arbitrary_component_device_matrix(thread_id);
		local_james_dean_data.call_set_arbitrary_component_device_matrix(local_arbitrary_component_device_matrix);
		local_arbitrary_component_device_matrix.clear();
	
		auto arbitrary_component_device_end_time = std::chrono::high_resolution_clock::now();
		auto arbitrary_component_device_elapsed_time = arbitrary_component_device_end_time - arbitrary_component_device_start_time;
		auto arbitrary_component_device_nanoseconds = std::chrono::duration_cast<std::chrono::nanoseconds>(arbitrary_component_device_elapsed_time);
		cpp_int arbitrary_component_device_printable = arbitrary_component_device_nanoseconds.count();
		mp_float arbitrary_component_device_float_50 = arbitrary_component_device_printable.convert_to<mp_float>();
		mp_float arbitrary_component_device_seconds = arbitrary_component_device_float_50 / 1'000'000'000;
		std::cout << "[" << thread_id << "] arbitrary_component_device (" << james_dean.arbitrary_component_device_target << ") creation time: " << arbitrary_component_device_seconds << "s\n";
		
		auto technical_matrix_start_time = std::chrono::high_resolution_clock::now();
		if(thread_id == 1)
		{
			std::cout << "[" << thread_id << "] creating technical matrix\n";
		}
		
		local_technical_matrix = return_technical_matrix(thread_id);
		local_james_dean_data.call_set_technical_matrix(local_technical_matrix);
		local_technical_matrix.clear();
		
		auto technical_matrix_end_time = std::chrono::high_resolution_clock::now();
		auto technical_matrix_elapsed_time = technical_matrix_start_time - technical_matrix_end_time;
		auto technical_matrix_nanoseconds = std::chrono::duration_cast<std::chrono::nanoseconds>(technical_matrix_elapsed_time);
		cpp_int technical_matrix_nanoseconds_printable = technical_matrix_nanoseconds.count();
		mp_float technical_matrix_float_50 = technical_matrix_nanoseconds_printable.convert_to<mp_float>();
		mp_float technical_matrix_seconds = technical_matrix_float_50 / 1'000'000'000;
		std::cout << "[" << thread_id << "] technical (" << james_dean.technical_target << ") creation time: " << technical_matrix_seconds << "ns\n";
		
		auto level_table_start_time = std::chrono::high_resolution_clock::now();
		if(thread_id == 1)
		{
			std::cout << "[" << thread_id << "] creating level_table\n";
		}
		
		local_level_table_vector = return_level_table(thread_id);
		local_james_dean_data.call_set_level_table(local_level_table_vector);
		local_level_table_vector.clear();
		
		auto level_table_end_time = std::chrono::high_resolution_clock::now();
		auto level_table_elapsed_time = level_table_end_time - level_table_start_time;
		auto level_table_nanoseconds = std::chrono::duration_cast<std::chrono::nanoseconds>(technical_matrix_elapsed_time);
		cpp_int level_table_nanoseconds_printable = level_table_nanoseconds.count();
		mp_float level_table_float = level_table_nanoseconds_printable.convert_to<mp_float>();
		mp_float level_table_seconds = level_table_float / 1'000'000'000;
		std::cout << "[" << thread_id << "] level_table (" << james_dean.level_table_target << ") creation_time: " << level_table_seconds << "\n";
		
		std::cout << "[" << thread_id << "] matrix creation completed for this thread (" << thread_id << ")\n";
		auto matrice_end_time = std::chrono::high_resolution_clock::now();
		auto matrice_elapsed_time = matrice_end_time - matrice_start_time;
		auto matrice_nanoseconds = std::chrono::duration_cast<std::chrono::nanoseconds>(matrice_elapsed_time);
		cpp_int matrice_nanoseconds_printable = matrice_nanoseconds.count();
		auto total_matrices_created = james_dean.data_buff_target + james_dean.positive_hex_target + james_dean.random_target + james_dean.random_target + james_dean.hash_table_target + james_dean.arbitrary_component_device_target;
		mp_float matrice_float_50 = matrice_nanoseconds_printable.convert_to<mp_float>();
		mp_float total_matrices_seconds = matrice_float_50 / 1'000'000'000;
		std::cout << "[" << thread_id << "] total matrice (" << total_matrices_created << ") creation time: " << total_matrices_seconds << "s\n";
		
		james_dean.local_james_dean_vector.push_back(&local_james_dean_data);
		
		james_dean.total_data_counter = james_dean.total_data_counter + 1;
		if(thread_id == 1)
		{
			std::cout << "[ " << thread_id << "] james_dean.local_james_dean_vector.size() = " << james_dean.local_james_dean_vector.size() << "\n";
		}
		
		james_dean.local_type_id = james_dean.local_type_id + 1;
		
		if(james_dean.local_type_id >= james_dean.type_target)
		{
			james_dean.local_type_id = 0;
			
			james_dean.local_class_id = james_dean.local_class_id + 1;
			
			if(james_dean.local_class_id >= james_dean.class_target)
			{
				james_dean.local_class_id = 0;
				
				james_dean.local_data_id = james_dean.local_data_id + 1;
				
				if(james_dean.local_data_id >= james_dean.data_target)
				{
					james_dean.local_data_id = 0;
					
					james_dean.local_production_id = james_dean.local_production_id + 1;
				}
			}
		}
		
		if(james_dean.local_production_id >= james_dean.production_target)
		{
			local_james_dean_vector = james_dean.local_james_dean_vector;
			
			while (james_dean.local_reserved_id < james_dean.world_population)
			{
				james_dean.local_reserved_id = james_dean.local_reserved_id + 200'000;
				
				std::cout << "[" << thread_id << "] reserves " << james_dean.local_reserved_id - 200'000 << " to " << james_dean.local_reserved_id << "\n";
				
				for(cpp_int current_reservation_id = 0; current_reservation_id < 200'000; current_reservation_id++)
				{
					civilian local_civilian;
					
					local_civilian.call_set_civilian_id(james_dean.local_civilian_id + current_reservation_id);
					
					local_civilian.call_set_civilian_james_dean_matrix(local_james_dean_vector);
				}
			}
		}		
	}
}

// ---=+=--- ---=+=--- ---=+=--- ---=+=--- ---=+=--- ---=+=--- ---=+=---
// RETURN DATA BUFF MATRIX

std::vector<std::vector<data_buff*>> return_data_buff_matrix(cpp_int thread_id)
{
	signed long long int current_db_vector = 0;
	
	std::vector<std::vector<data_buff*>> local_data_buff_matrix;
	std::vector<data_buff*> small_data_buff_vector;
	local_data_buff_matrix.push_back(small_data_buff_vector);
	
	std::vector<std::vector<cpp_int>> local_random_matrix;
	std::vector<std::vector<cpp_int>> local_mod_random_matrix;
	
	std::vector<std::vector<hash_table*>> local_hash_table_matrix;
	
	cpp_int data_buff_target = 10;
	cpp_int data_buff_count = 0;
	cpp_int data_buff_counter = 0;
	
	while (data_buff_count < data_buff_target)
	{
		if(thread_id == 1)
		{
			std::cout << "[" << thread_id << "] data_buff_counter = " << data_buff_counter << " of " << data_buff_target << "\n";
		}
		
		data_buff local_data_buff;
		
		local_data_buff.call_set_local_system_id(james_dean.local_system_id);
		
		james_dean.local_system_id = james_dean.local_system_id + 1;
		
		local_data_buff.call_set_local_production_id(james_dean.local_production_id);
		
		local_data_buff.call_set_local_data_id(james_dean.local_data_id);
		
		local_data_buff.call_set_local_class_id(james_dean.local_class_id);
		
		local_data_buff.call_set_local_type_id(james_dean.local_type_id);
		
		james_dean.local_type_id = james_dean.local_type_id + 1;
		
		if(james_dean.local_type_id >= james_dean.type_target)
		{
			james_dean.local_type_id = 0;
			
			james_dean.local_class_id = james_dean.local_class_id + 1;
			
			if(james_dean.local_class_id >= james_dean.class_target)
			{
				james_dean.local_class_id = 0;
				
				james_dean.local_data_id = james_dean.local_data_id + 1;
				
				if(james_dean.local_data_id >= james_dean.data_target)
				{
					james_dean.local_data_id = 0;
					
					james_dean.local_production_id = james_dean.local_production_id + 1;
				}
			}
		}
		
		local_random_matrix = return_random_matrix(thread_id);
		local_data_buff.call_set_local_random_matrix(local_random_matrix);
		local_random_matrix.clear();
		
		local_mod_random_matrix = return_random_matrix(thread_id);
		local_data_buff.call_set_local_mod_random_matrix(local_mod_random_matrix);
		local_mod_random_matrix.clear();
		
		/*
		local_hash_table_matrix = return_hash_table_matrix(thread_id);
		local_data_buff.call_set_local_hash_table_matrix(local_hash_table_matrix);
		local_hash_table_matrix.clear();
		*/
		
		local_data_buff_matrix[current_db_vector].push_back(&local_data_buff);
		
		if(local_data_buff_matrix[current_db_vector].size() >= data_buff_target)
		{
			current_db_vector = current_db_vector + 1;
			
			local_data_buff_matrix.push_back(small_data_buff_vector);
		}
		
		data_buff_count = data_buff_count + 1;
		data_buff_counter = data_buff_counter + 1;
	}
	
	return(local_data_buff_matrix);
}

// ---=+=--- ---=+=--- ---=+=--- ---=+=--- ---=+=--- ---=+=--- ---=+=---
// RETURN POSITIVE HEX MATRIX

std::vector<std::vector<positive_hex*>> return_positive_hex_matrix(cpp_int thread_id)
{
	signed long long int current_ph_vector = 0;
	
	std::vector<std::vector<positive_hex*>> local_positive_hex_matrix;
	std::vector<positive_hex*> small_positive_hex_vector;
	local_positive_hex_matrix.push_back(small_positive_hex_vector);
	
	std::vector<std::vector<cpp_int>> local_random_matrix;
	std::vector<std::vector<cpp_int>> local_mod_random_matrix;
	
	std::vector<std::vector<hash_table*>> local_hash_table_matrix;
	
	cpp_int positive_hex_count = 0;
	cpp_int positive_hex_target = 10;
	
	while (positive_hex_count < positive_hex_target)
	{
		if(thread_id == 1)
		{
			std::cout << "[" << thread_id << "] positive_hex " << positive_hex_count << " of " << positive_hex_target << "\n";
		}
		
		positive_hex local_positive_hex;
		
		local_positive_hex.call_set_local_system_id(james_dean.local_system_id);
		james_dean.local_system_id = james_dean.local_system_id + 1;
		
		local_positive_hex.call_set_local_production_id(james_dean.local_production_id);
		
		local_positive_hex.call_set_local_data_id(james_dean.local_data_id);
		
		local_positive_hex.call_set_local_class_id(james_dean.local_class_id);
		
		local_positive_hex.call_set_local_type_id(james_dean.local_type_id);
		
		james_dean.local_type_id = james_dean.local_type_id + 1;
		
		if(james_dean.local_type_id >= james_dean.type_target)
		{
			james_dean.local_type_id = 0;
			
			james_dean.local_class_id = james_dean.local_class_id + 1;
			
			if(james_dean.local_class_id >= james_dean.class_target)
			{
				james_dean.local_class_id = 0;
				
				james_dean.local_data_id = james_dean.local_data_id + 1;
				
				if(james_dean.local_data_id >= james_dean.data_target)
				{
					james_dean.local_data_id = 0;
					
					james_dean.local_production_id = james_dean.local_production_id + 1;
				}
			}
		}
		
		local_random_matrix = return_random_matrix(thread_id);
		local_positive_hex.call_set_local_random_matrix(local_random_matrix);
		local_random_matrix.clear();
		
		local_mod_random_matrix = return_random_matrix(thread_id);
		local_positive_hex.call_set_local_mod_random_matrix(local_mod_random_matrix);
		local_mod_random_matrix.clear();
		
		/*
		local_hash_table_matrix = return_hash_table_matrix(thread_id);
		local_positive_hex.call_set_local_hash_table_matrix(local_hash_table_matrix);
		local_hash_table_matrix.clear();
		*/
		
		local_positive_hex_matrix[current_ph_vector].push_back(&local_positive_hex);
		
		if(local_positive_hex_matrix[current_ph_vector].size() > 30)
		{
			current_ph_vector = current_ph_vector + 1;
			
			local_positive_hex_matrix.push_back(small_positive_hex_vector);
		}
		
		positive_hex_count = positive_hex_count + 1;
	}
	
	return(local_positive_hex_matrix);
}

// ---=+=--- ---=+=--- ---=+=--- ---=+=--- ---=+=--- ---=+=--- ---=+=---
// RETURN ARBITRARY COMPONENT DEVICE

std::vector<std::vector<arbitrary_component_device*>> return_arbitrary_component_device_matrix(cpp_int thread_id)
{
	signed long long int current_acd_vector = 0;
	
	std::vector<std::vector<cpp_int>> local_random_matrix;
	std::vector<std::vector<cpp_int>> local_mod_random_matrix;
	
	std::vector<std::vector<hash_table*>> local_hash_table_matrix;
	
	std::vector<std::vector<arbitrary_component_device*>> local_arbitrary_component_device_matrix;
	std::vector<arbitrary_component_device*> small_arbitrary_component_device_vector;
	local_arbitrary_component_device_matrix.push_back(small_arbitrary_component_device_vector);
	
	while (james_dean.arbitrary_component_device_count < james_dean.arbitrary_component_device_target)
	{
		if(thread_id == 1)
		{
			std::cout << "[" << thread_id << "] thread_id(1) reports acd_count at " << james_dean.arbitrary_component_device_count << "\n";
		}
		
		arbitrary_component_device local_arbitrary_component_device;
		
		local_arbitrary_component_device.call_set_local_system_id(james_dean.local_system_id);
		
		james_dean.local_system_id = james_dean.local_system_id + 1;
		
		local_arbitrary_component_device.call_set_local_production_id(james_dean.local_production_id);
		
		local_arbitrary_component_device.call_set_local_data_id(james_dean.local_data_id);
		
		local_arbitrary_component_device.call_set_local_class_id(james_dean.local_class_id);
		
		local_arbitrary_component_device.call_set_local_type_id(james_dean.local_type_id);
		
		james_dean.local_type_id = james_dean.local_type_id + 1;
		
		if(james_dean.local_type_id >= james_dean.type_target)
		{
			james_dean.local_type_id = 0;
			
			james_dean.local_class_id = james_dean.local_class_id + 1;
			
			if(james_dean.local_class_id >= james_dean.class_target)
			{
				james_dean.local_class_id = 0;
				
				james_dean.local_data_id = james_dean.local_data_id + 1;
				
				if(james_dean.local_data_id >= james_dean.data_target)
				{
					james_dean.local_data_id = 0;
					
					james_dean.local_production_id = james_dean.local_production_id + 1;
				}
			}
		}
		
		local_random_matrix = return_random_matrix(thread_id);
		local_arbitrary_component_device.call_set_local_random_matrix(local_random_matrix);
		local_random_matrix.clear();
		
		local_mod_random_matrix = return_random_matrix(thread_id);
		local_arbitrary_component_device.call_set_local_mod_random_matrix(local_mod_random_matrix);
		local_mod_random_matrix.clear();
		
		/*
		local_hash_table_matrix = return_hash_table_matrix(thread_id);
		local_arbitrary_component_device.call_set_local_hash_table_matrix(local_hash_table_matrix);
		local_hash_table_matrix.clear();
		*/
		
		local_arbitrary_component_device_matrix[current_acd_vector].push_back(&local_arbitrary_component_device);
		
		if(local_arbitrary_component_device_matrix[current_acd_vector].size() > 30)
		{
			current_acd_vector = current_acd_vector + 1;
			
			local_arbitrary_component_device_matrix.push_back(small_arbitrary_component_device_vector);
		}
		
		james_dean.arbitrary_component_device_count = james_dean.arbitrary_component_device_count + 1;
	}
	
	return(local_arbitrary_component_device_matrix);
}

// ---=+=--- ---=+=--- ---=+=--- ---=+=--- ---=+=--- ---=+=--- ---=+=---
// RETURN HASH TABLE MATRIX

std::vector<std::vector<hash_table*>> return_hash_table_matrix(cpp_int thread_id)
{
    std::string alphabet_str = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ";

    // from random_device called seed_source
    pcg_extras::seed_seq_from<std::random_device> seed_source;

    // using pcg64_k1024, using seed_source
    pcg32_k16384 pcg_k16(seed_source);
    std::uniform_int_distribution<> uniform_dist(0, alphabet_str.size());
    signed long long int random_result = uniform_dist(pcg_k16);

    cpp_int string_size = 50;
    cpp_int table_size = 50;
    cpp_int vector_count = 50;

    cpp_int table_counter = 0;
    cpp_int str_counter = 0;

    std::vector<std::string> local_hash_table;
    std::vector<std::vector<std::string>> local_hash_vector;
    std::string local_str = "void";
    local_str.clear();
    
    std::string str_from = "void";
    str_from.clear();

    std::vector<std::vector<std::string*>> local_big_vector;
    std::vector<std::string*> local_small_vector;

    std::vector<std::vector<hash_table*>> hash_table_vector;
    std::vector<hash_table*> small_hash_table_vector;

    cpp_int hash_table_target = 4;
    cpp_int vector_target = 7;
    cpp_int table_target = 5;
    cpp_int str_target = 6;
    cpp_int letter_target = 10;
    
    cpp_int set_str_target = 5;
    
    std::vector<std::string> local_str_vector;
    
    std::vector<std::vector<cpp_int>> local_random_matrix;
    std::vector<std::vector<cpp_int>> local_mod_random_matrix;
    
    cpp_int display_counter = 0;
    
    for(signed long long int current_hash_vector = 0; current_hash_vector < hash_table_target; current_hash_vector++)
    {
    	hash_table_vector.push_back(small_hash_table_vector);
    	
    	for(signed long long int current_hash_table = 0; current_hash_table < table_target; current_hash_table++)
    	{   		
    		for(signed long long int current_vector = 0; current_vector < vector_target; current_vector++)
    		{
    			if(thread_id == 1)
    			{
    				display_counter = display_counter + 1;
    				
    				if(display_counter > 20)
    				{
    					display_counter = 0;
    					
    					std::cout << "[" << thread_id << "] hash_table_vector " << current_vector << "/" << vector_target << "::" << current_hash_table << "/" << table_target << "::" << current_hash_vector << "/" << hash_table_target << "\n";
    				}
    			}
		
				local_big_vector.push_back(local_small_vector);
		
				for(signed long long int current_small_vector = 0; current_small_vector < table_target; current_small_vector++)
				{
					for(signed long long int current_str_set = 0; current_str_set < set_str_target; current_str_set++)
					{
						for(signed long long int current_str = 0; current_str < str_target; current_str++)
						{
							for(signed long long int current_letter = 0; current_letter < letter_target; current_letter++)
							{
								random_result = uniform_dist(pcg_k16);
						
								str_from = alphabet_str[random_result];
						
								local_str = local_str + str_from;
							}
					
							// we now have the string...
					
							local_big_vector[current_vector].push_back(&local_str);
					
						}
				
						// now we have the strings inside of small vectors
				
						// I guess do nothing here
				
					}
			
				}
			}
			
			hash_table local_hash_table;
			
			local_hash_table.call_set_local_hash_table(local_big_vector);
			local_big_vector.clear();
			
			local_hash_table.call_set_local_production_id(james_dean.local_production_id);
			local_hash_table.call_set_local_data_id(james_dean.local_data_id);
			local_hash_table.call_set_local_class_id(james_dean.local_class_id);
			local_hash_table.call_set_local_type_id(james_dean.local_type_id);
			
			james_dean.local_type_id = james_dean.local_type_id + 1;
			
			if(james_dean.local_type_id >= james_dean.type_target)
			{
				james_dean.local_type_id = 0;
				
				james_dean.local_class_id = james_dean.local_class_id + 1;
				
				if(james_dean.local_class_id >= james_dean.class_target)
				{
					james_dean.local_class_id = 0;
					
					james_dean.local_data_id = james_dean.local_data_id + 1;
					
					if(james_dean.local_data_id >= james_dean.data_target)
					{
						james_dean.local_data_id = 0;
						
						james_dean.local_production_id = james_dean.local_production_id + 1;
					}
				}
			}
			
			hash_table_vector[current_hash_vector].push_back(&local_hash_table);
		}
	}
    
    	
    return(hash_table_vector);
}

// ---=+=--- ---=+=--- ---=+=--- ---=+=--- ---=+=--- ---=+=--- ---=+=---
// RETURN RANDOM MATRIX

std::vector<std::vector<cpp_int>> return_random_matrix(cpp_int thread_id)
{
	pcg_extras::seed_seq_from<std::random_device> seed_source;
    pcg32_k16384 pcg_k16(seed_source);

    cpp_int vector_target = 70;
    cpp_int set_target = 40;
    cpp_int random_target = 200;

    std::vector<std::vector<cpp_int>> local_big_random_vector;
    std::vector<cpp_int> local_small_random_vector;

    cpp_int vector_counter = 0;
    cpp_int set_counter = 0;
    cpp_int random_counter = 0;

    cpp_int random_number = 0;

    for(signed long long int current_vector = 0; current_vector < vector_target; current_vector++)
    {
        local_big_random_vector.push_back(local_small_random_vector);

        for(signed long long int current_set = 0; current_set < set_target; current_set++)
        {
            for(signed long long int current_number = 0; current_number < random_target; current_number++)
            {
                random_number = pcg_k16();
                
                random_number = random_number * random_number * random_number; // tida power of 3

                local_big_random_vector[current_vector].push_back(random_number);
            }
        }
    }

    return(local_big_random_vector);
}

// ---=+=--- ---=+=--- ---=+=--- ---=+=--- ---=+=--- ---=+=--- ---=+=---
// RETURN TECHNICAL MATRIX

std::vector<std::vector<technical*>> return_technical_matrix(cpp_int thread_id)
{
	cpp_int local_technical_data_counter = 0;
	cpp_int technical_matrix_main_vector_target = 4;
	cpp_int technical_matrix_small_vector_target = 7;
	cpp_int technical_matrix_target = technical_matrix_main_vector_target * technical_matrix_small_vector_target;
	signed long long int current_technical_data_vector = 0;
	
	std::vector<std::vector<technical*>> technical_matrix;
	std::vector<technical*> small_technical_matrix;
	technical_matrix.push_back(small_technical_matrix);
	
	std::vector<std::vector<cpp_int>> local_random_matrix;
	std::vector<std::vector<cpp_int>> local_mod_random_matrix;
	
	std::vector<std::vector<positive_hex*>> local_positive_hex_matrix;
	std::vector<std::vector<data_buff*>> local_data_buff_matrix;
	std::vector<std::vector<hash_table*>> local_hash_table_matrix;
	
	signed long long int technical_matrix_counter = 0;
	
	while (technical_matrix_counter < technical_matrix_target)
	{
		if(thread_id == 1)
		{
			std::cout << "[" << thread_id << "] technical_vector " << technical_matrix_counter << "/" << technical_matrix_target << "\n";
		}
    	
		technical local_technical_data;
		
		local_technical_data.call_set_local_system_id(james_dean.local_system_id);
		
		james_dean.local_system_id = james_dean.local_system_id + 1;
		
		local_technical_data.call_set_local_production_id(james_dean.local_production_id);
		
		local_technical_data.call_set_local_data_id(james_dean.local_data_id);
		
		local_technical_data.call_set_local_class_id(james_dean.local_class_id);
		
		local_technical_data.call_set_local_type_id(james_dean.local_type_id);
		
		james_dean.local_type_id = james_dean.local_type_id + 1;
		
		if(james_dean.local_type_id >= james_dean.type_target)
		{
			james_dean.local_type_id = 0;
			
			james_dean.local_class_id = james_dean.local_class_id + 1;
			
			if(james_dean.local_class_id >= james_dean.class_target)
			{
				james_dean.local_class_id = 0;
				
				james_dean.local_data_id = james_dean.local_data_id + 1;
				
				if(james_dean.local_data_id >= james_dean.data_target)
				{
					james_dean.local_data_id = 0;
					
					james_dean.local_production_id = james_dean.local_production_id + 1;
				}
			}
		}
		
		local_random_matrix = return_random_matrix(thread_id);
		local_technical_data.call_set_local_random_matrix(local_random_matrix);
		local_random_matrix.clear();
		
		local_mod_random_matrix = return_random_matrix(thread_id);
		local_technical_data.call_set_local_mod_random_matrix(local_mod_random_matrix);
		local_mod_random_matrix.clear();
		
		/*
		following 3 are commented out as computer isn't fast enough to make them, /they are made wrong?
		
		local_positive_hex_matrix = return_positive_hex_matrix(thread_id);
		local_technical_data.call_set_local_positive_hex_matrix(local_positive_hex_matrix);
		local_positive_hex_matrix.clear();
		
		local_data_buff_matrix = return_data_buff_matrix(thread_id);
		local_technical_data.call_set_local_data_buff_matrix(local_data_buff_matrix);
		local_data_buff_matrix.clear();
		*/
		
		/*
		local_hash_table_matrix = return_hash_table_matrix(thread_id);
		local_technical_data.call_set_local_hash_table_matrix(local_hash_table_matrix);
		local_hash_table_matrix.clear();
		*/
		
		technical_matrix[current_technical_data_vector].push_back(&local_technical_data);
		
		local_technical_data_counter = local_technical_data_counter + 1;
		technical_matrix_counter = technical_matrix_counter + 1;
		
		if(local_technical_data_counter >= technical_matrix_small_vector_target)
		{
			local_technical_data_counter = 0;
			
			current_technical_data_vector = current_technical_data_vector + 1;
			
			technical_matrix.push_back(small_technical_matrix);
		}
	}
	
	return(technical_matrix);
}

// ---=+=--- ---=+=--- ---=+=--- ---=+=--- ---=+=--- ---=+=--- ---=+=---
// RETURN LEVEL TABLE

std::vector<level_table*> return_level_table(cpp_int thread_id)
{
	std::vector<level_table*> local_level_table_matrix;
	
	pcg_extras::seed_seq_from<std::random_device> seed_source;
    pcg32_k16384 pcg_k16(seed_source);
    
    cpp_int random_number = pcg_k16();
    
    std::vector<std::vector<cpp_int>> local_table_matrix;
    std::vector<std::vector<std::vector<cpp_int>>> local_table_matrices;
	std::vector<cpp_int> small_table;
	
	cpp_int table_target = 10;
	cpp_int set_target = 12;
	cpp_int vector_target = 10;
	cpp_int number_target = 30;
	
	cpp_int level_table_target = 9;
	
	for(signed long long int level_table_counter = 0; level_table_counter < level_table_target; level_table_counter++)
	{
		if(thread_id == 1)
		{
			std::cout << "[" << thread_id << "] level_table_vector " << level_table_counter << "/" << level_table_target << "\n";
		}
    	
		for(signed long long int current_big_table = 0; current_big_table < table_target; current_big_table++)
		{
			local_table_matrices.push_back(local_table_matrix);
		
			for(signed long long int current_small_table = 0; current_small_table < set_target; current_small_table++)
			{
				local_table_matrices[current_big_table].push_back(small_table);
			
				for(signed long long int current_number = 0; current_number < number_target; current_number++)
				{
					random_number = pcg_k16();
				
					local_table_matrices[current_big_table][current_small_table].push_back(random_number);
				}
			}
		}
		
		level_table local_level_table;
    
    	local_level_table.call_set_level_table_matrix(local_table_matrix);
    	local_level_table.call_set_table_matrices(local_table_matrices);
    
    	local_level_table_matrix.push_back(&local_level_table);
    }
    
    return(local_level_table_matrix);
}

// --==++ MARATHON


	

1. Create the root partition. Decide how and where your root "/" partition will be, you cannot have two partitions that have the "root" piece on them, so pick how much of a size your drive will be, I always go for 200gb to 300gb for my 4tb drive, but a 1tb drive may be the same as well. If your going to install intel's programming tools then you'll need the 250gb as they are hefty.
2. The version of Ubuntu/Debian may make a /boot/efi partition. If not, make a partition with the mount point of "/boot/efi" and have it to be 24gb. I do not know if you need one that big, but I stay on the safe side.
3. Make a /boot mounted partition to be 24gb
4. Make a swap partiion of 24gb
5. Make the rest of your space to be /home

Leveraging Multi-Core Architectures: A Guide to Python's multiprocessing Module for 24-Process Parallelism
1. Introduction: Harnessing Parallel Processing in Python

The proliferation of multi-core processors in modern computing systems presents both an opportunity and a challenge for software developers. To fully exploit the computational power available, applications must be designed to execute tasks in parallel. Python, despite some ofits inherent characteristics, provides robust mechanisms for achieving parallelism. The user query specifically requests a Python program employing "24-threads using the multiprocessing module." It is crucial to clarify at the outset that the multiprocessing module in Python is designed to create and manage processes, not threads in the traditional shared-memory sense. While Python's threading module does manage threads, its ability to achieve true parallelism for CPU-bound tasks is constrained by the Global Interpreter Lock (GIL).

The multiprocessing package, a part of Python's standard library since version 2.6, offers a path to true parallelism by utilizing subprocesses instead of threads, thereby effectively sidestepping the GIL. Each subprocess spawned by multiprocessing has its own Python interpreter and memory space, allowing them to run concurrently on different CPU cores. This report aims to provide a comprehensive guide to using Python's multiprocessing module, specifically demonstrating how to construct a program that leverages 24 parallel processes. Beyond the example program, this document will delve into the fundamental concepts, best practices, and advanced considerations essential for effective parallel programming in Python.  

2. Processes vs. Threads: The Global Interpreter Lock (GIL) and Its Ramifications

Understanding the distinction between processes and threads, particularly in the context of Python's Global Interpreter Lock (GIL), is fundamental to designing efficient parallel applications.
Conceptual Distinction

Processes, as managed by the multiprocessing module, are independent execution units. Each process possesses its own memory space and its own Python interpreter. This isolation means that processes do not share data by default and communication between them must be explicit, typically through mechanisms like pipes or queues. This model allows processes to run truly in parallel on multi-core systems.  

Threads, managed by Python's threading module, exist within a single process and share the same memory space. While this shared memory allows for easier data exchange between threads, it also introduces the need for careful synchronization to prevent race conditions. More critically for Python, threads are subject to the Global Interpreter Lock.  

The Global Interpreter Lock (GIL)

The GIL is a mutex (a type of lock) that protects access to Python objects, preventing multiple native threads from executing Python bytecode at the same time within a single process. This lock must be held by a thread before it can safely access Python objects or call Python/C API functions.  

    Impact on CPU-bound tasks: For tasks that are computationally intensive (CPU-bound), the GIL becomes a significant bottleneck. Even on a multi-core processor, only one Python thread can execute Python bytecode at any given moment. Consequently, using the threading module for CPU-bound tasks typically results in concurrency (tasks taking turns on a single core, giving the appearance of parallelism) rather than true parallelism (tasks running simultaneously on multiple cores). This can mean that a multithreaded CPU-bound Python program may not run significantly faster, or could even run slower, than a single-threaded version due to the overhead of thread management and GIL contention. In contrast, multiprocessing bypasses the GIL by using separate processes, each with its own interpreter, enabling true parallelism for CPU-bound workloads.   

Impact on I/O-bound tasks: For tasks that spend much of their time waiting for external operations to complete (I/O-bound), such as network requests or file system operations, the threading module can still be effective. The GIL is often released by a thread during these blocking I/O calls, allowing other threads within the same process to run. This allows for concurrent execution of I/O operations, improving responsiveness and throughput.  

The GIL was implemented in CPython for several reasons, including simplifying memory management and making it easier to write C extensions for Python. CPython's memory management is not inherently thread-safe, and the GIL provides a straightforward mechanism to ensure that only one thread is manipulating Python objects at a time, thus preventing race conditions at the C level and simplifying the reference counting scheme. While this design choice simplifies CPython's internals and the development of C extensions, its primary consequence is the limitation on multithreaded parallelism for CPU-bound tasks. This historical design decision has led to multiprocessing becoming the standard Python solution for leveraging multiple CPU cores for computationally intensive work.  

Choosing multiprocessing for CPU-Intensive Work

Given that the user's request for "24-threads" likely implies a desire to maximize CPU utilization for potentially intensive tasks, the multiprocessing module is the appropriate choice. It allows the creation of 24 independent processes, each capable of running on a separate CPU core if available, thus achieving true parallelism and overcoming the limitations imposed by the GIL on CPU-bound operations.  

The following table provides a comparative overview of multiprocessing and threading:

Table 1: multiprocessing vs. threading - A Comparative Overview
Feature	multiprocessing (Processes)	threading (Threads)
Mechanism	Separate processes, each with its own Python interpreter	Threads within a single process, sharing one interpreter
Memory Space	Independent memory space per process	Shared memory space within the process
GIL Impact	Bypassed; allows true parallelism on multi-core systems	Limited by GIL; typically achieves concurrency, not parallelism for CPU-bound tasks
CPU-bound Tasks	Preferred; enables full utilization of multiple CPU cores	Limited speedup due to GIL; may even slow down performance
I/O-bound Tasks	Can be used, but may have higher overhead than threads	Often efficient as GIL is released during I/O waits
Overhead	Higher (process creation, inter-process communication)	Lower (thread creation, shared memory access)
Communication	Requires explicit Inter-Process Communication (IPC) mechanisms	Direct data sharing (requires synchronization primitives)
3. Implementing 24-Process Parallelism with multiprocessing.Pool

The multiprocessing.Pool class offers a convenient and high-level interface for distributing tasks across a pool of worker processes. This is often the simplest way to parallelize the execution of a function across multiple input values.
The multiprocessing.Pool Class

A Pool object controls a pool of worker processes to which jobs can be submitted. It manages the distribution of tasks to these workers and can also handle the collection of results. The number of worker processes in the pool is specified by the processes argument during instantiation; for this report's objective, this would be 24. Using the Pool as a context manager (i.e., with a with statement) is recommended as it ensures the proper termination and cleanup of worker processes.  

Submitting Tasks to the Pool

Several methods are available for submitting tasks:

    Pool.map(func, iterable, chunksize=None): This method is analogous to the built-in map() function. It applies the function func to every item in iterable, distributing the work among the worker processes. It blocks until all tasks are completed and returns a list of results in the same order as the input iterable. The optional chunksize argument can optimize performance for very long iterables by sending work to worker processes in larger batches.   

Pool.apply_async(func, args=(), kwds={}, callback=None, error_callback=None): This method submits a single task (func with args and kwds) to a worker process asynchronously. It returns an AsyncResult object immediately, which can be used to query the status of the task and retrieve its result later. Optional callback and error_callback functions can be provided to handle results or errors.  

    Pool.starmap(func, iterable_of_tuples): Similar to Pool.map(), but the elements of iterable_of_tuples are expected to be iterables that are unpacked as arguments for func. This is useful when the worker function takes multiple arguments.

Gathering Results

The method for gathering results depends on how tasks were submitted:

    For Pool.map() and Pool.starmap(), the methods themselves block until all results are available and return them as a list.
    For Pool.apply_async(), the returned AsyncResult object has several methods:
        get(timeout=None): Waits for the result to become available and returns it. If a timeout is specified and the result does not arrive within that time, a multiprocessing.TimeoutError is raised. If the target function raised an exception, get() will re-raise that exception in the main process.   

        ready(): Returns True if the task has completed, False otherwise.
        successful(): Returns True if the task completed without raising an exception (can only be called if ready() is True).

The if __name__ == '__main__': Guard

When using multiprocessing, particularly on platforms like Windows that use the 'spawn' start method by default, it is crucial to protect the main part of the program that launches new processes within an if __name__ == '__main__': block. When a new process is spawned, it needs to import the main module. Without this guard, the process creation code would be executed again in the child process, leading to recursive spawning of processes and likely a RuntimeError. This guard ensures that processes are only created when the script is executed directly, not when it is imported by a child process.  

Example Python Program: 24-Process Computation

The following Python program demonstrates the use of multiprocessing.Pool to execute a simulated CPU-bound task across 24 worker processes.
Python

import multiprocessing
import os
import time

def worker_function(item_id, duration):
    """
    Simulates a CPU-bound task.
    Each task takes a specific item_id and a simulated duration.
    """
    # Simulate a CPU-bound task
    print(f"Process {os.getpid()} starting task {item_id}.")
    start_time = time.time()
    # Example computation: loop scaled by duration
    # The number 1e7 is arbitrary; adjust based on machine speed for noticeable time.
    # For very fast machines or very short durations, this loop might still be too quick.
    # The goal is to make each task last for a fraction of a second to a few seconds.
    for _ in range(int(duration * 1e7)): 
        pass # Intensive computation placeholder
    end_time = time.time()
    result = f"Task {item_id} completed by process {os.getpid()} in {end_time - start_time:.2f} seconds."
    print(f"Process {os.getpid()} finished task {item_id}.")
    return result

if __name__ == '__main__':
    NUMBER_OF_PROCESSES = 24
    
    # Prepare some tasks (e.g., a list of items to process)
    # Each task is a tuple (item_id, processing_duration_simulation)
    # Here, we create 48 tasks to ensure each of the 24 processes gets work.
    # Durations vary to simulate heterogeneous tasks.
    tasks_to_process = 

    print(f"Main process {os.getpid()} starting.")
    print(f"Creating a Pool of {NUMBER_OF_PROCESSES} worker processes.")

    start_overall_time = time.time()

    # Using a context manager for the Pool ensures it's properly closed.
    with multiprocessing.Pool(processes=NUMBER_OF_PROCESSES) as pool:
        # Using starmap to pass multiple arguments (item_id, duration) 
        # from each tuple in tasks_to_process to worker_function.
        # starmap blocks until all tasks are complete and returns results in order.
        results = pool.starmap(worker_function, tasks_to_process)

        # --- Alternative using apply_async for more control or heterogeneous tasks ---
        # This approach is useful if you want to process results as they complete
        # or if tasks have different error handling needs or callbacks.
        #
        # async_results_objects =
        # for item_id, duration in tasks_to_process:
        #     res_obj = pool.apply_async(worker_function, args=(item_id, duration))
        #     async_results_objects.append(res_obj)
        #
        # results =
        # print("Collecting results from async tasks...")
        # for i, res_obj in enumerate(async_results_objects):
        #     try:
        #         # Wait for each result, with a timeout (e.g., 60 seconds)
        #         results.append(res_obj.get(timeout=60)) 
        #     except multiprocessing.TimeoutError:
        #         print(f"Task for item_id (approx {tasks_to_process[i]}) timed out.")
        #         results.append(f"Error: Task for item_id (approx {tasks_to_process[i]}) timed out.")
        #     except Exception as e:
        #         print(f"Task for item_id (approx {tasks_to_process[i]}) failed with error: {e}")
        #         results.append(f"Error: Task for item_id (approx {tasks_to_process[i]}) failed: {e}")
        # --- End of alternative ---

    end_overall_time = time.time()

    print("\n--- All tasks completed ---")
    for res_text in results:
        if res_text: 
            print(res_text)

    print(f"\nTotal execution time with {NUMBER_OF_PROCESSES} processes: {end_overall_time - start_overall_time:.2f} seconds.")
    print(f"Main process {os.getpid()} finished.")

Explanation of the Program:

    Imports: multiprocessing for parallel execution, os to get process IDs (PIDs), and time for timing operations.
    worker_function(item_id, duration): This function simulates a CPU-bound task. It takes an item_id for identification and a duration to scale a computational loop. It prints messages upon starting and finishing and returns a string summarizing its execution. The os.getpid() call demonstrates that different tasks are handled by different processes.
    if __name__ == '__main__': block: This essential guard ensures that the code within it only runs when the script is executed directly.
    NUMBER_OF_PROCESSES = 24: Sets the desired number of worker processes.
    tasks_to_process: A list of tuples is created, where each tuple (item_id, duration) represents a task. In this example, 48 tasks are generated with varying simulated durations.
    multiprocessing.Pool Creation: A Pool of 24 worker processes is created using with multiprocessing.Pool(processes=NUMBER_OF_PROCESSES) as pool:. The with statement ensures the pool is properly closed and worker processes are joined when the block exits.
    Task Submission with pool.starmap: The pool.starmap(worker_function, tasks_to_process) call distributes the worker_function across the tasks_to_process. For each tuple (item_id, duration) in tasks_to_process, worker_function(item_id, duration) is called in one of the pool's worker processes. starmap conveniently unpacks these tuples into arguments for the worker function. It blocks until all tasks are complete.
    Alternative with apply_async (commented out): The commented section shows how pool.apply_async could be used. This method is non-blocking and returns AsyncResult objects. A loop would then be needed to call get() on each of these objects to retrieve the results, often with error handling and timeouts.
    Result Collection: The results list will contain the return values from each call to worker_function, in the order corresponding to tasks_to_process.
    Timing: The overall execution time is measured and printed to demonstrate the performance benefits of parallelism.

The creation of processes and the communication of data between them (arguments to workers, return values) inherently incur overhead. Each process, especially when using 'spawn' or 'forkserver' start methods, requires setting up a new memory space and potentially a new Python interpreter. Furthermore, data passed to worker functions and results returned must be serialized (typically using pickle) and then deserialized in the respective process. This serialization/deserialization can be time-consuming, particularly for large or complex data objects.  

If the tasks assigned to worker processes are too small (i.e., they complete very quickly, exhibiting low granularity), the overhead associated with managing the parallelism—such as process creation, inter-process communication (IPC), and data serialization—can outweigh the benefits of executing tasks in parallel. In such scenarios, a multiprocessing program might even run slower than its sequential counterpart. Conversely, if tasks are excessively large and vary significantly in execution time, load balancing across the worker processes might become inefficient, with some processes finishing much earlier than others and sitting idle. Therefore, it is crucial to design the worker_function to perform a substantial enough amount of computation to justify the parallelization overhead. Profiling the application is often necessary to determine an optimal task granularity. The chunksize argument in Pool.map and Pool.starmap can also be beneficial by allowing each worker process to receive a larger batch of items to process at once, thereby reducing the frequency of IPC and potentially improving efficiency for iterables with many small items.  

4. Orchestrating Data: Inter-Process Communication and Serialization

When processes run independently, mechanisms are needed if they must exchange data during their execution, beyond the initial arguments passed to worker functions and the final results collected by the main process. Python's multiprocessing module provides several tools for Inter-Process Communication (IPC) and manages the necessary data serialization.
Message Passing with Queue and Pipe

    multiprocessing.Queue: This class implements a process-safe and thread-safe First-In, First-Out (FIFO) queue. It is a common way to pass messages or task data between processes. Any Python object that can be pickled can be put onto a Queue using q.put() and retrieved by another process using q.get(). For example, a main process might populate a queue with tasks, and worker processes would retrieve and process these tasks. Similarly, workers could put their results onto another queue for the main process to collect.   

multiprocessing.Pipe(): The Pipe() function returns a pair of Connection objects connected by an underlying OS pipe. By default, pipes are duplex, meaning they allow two-way communication. Each Connection object has send() and recv() methods for transmitting picklable Python objects. Pipes are generally suitable for communication between two specific processes, such as a parent and a child it spawns, or between two worker processes if their connection objects are appropriately shared.  

Shared State Management: Manager Objects, Value, and Array

For scenarios where processes need to operate on common, mutable data structures, multiprocessing offers mechanisms for shared state.

    multiprocessing.Manager(): A call to Manager() starts a separate server process. This server process can host various Python objects (like lists, dictionaries, locks, semaphores, etc.), and the Manager returns proxy objects that other processes can use to interact with these shared objects. Modifications made via these proxies are reflected in the server process and are visible to all other processes holding proxies to the same shared object. For instance, manager.list() creates a shared list. While convenient for sharing complex Python data types, Manager objects are generally slower than direct shared memory due to the overhead of communication with the server process and the proxy object layer.   

multiprocessing.Value and multiprocessing.Array: These tools allow data to be stored in a shared memory map that can be accessed by multiple processes.  

        Value(typecode, initial_value, lock=True) creates a shared object for a single value of a basic C type (e.g., 'i' for integer, 'd' for double-precision float).
        Array(typecode, size_or_initializer, lock=True) creates a shared array of basic C type elements. By default, access to these shared memory objects is protected by a lock, which can be disabled if manual synchronization is preferred. Value and Array are more efficient for simple data types because they avoid the pickling overhead associated with Queue, Pipe, or Manager objects for each access. However, they are limited to basic C types and require careful synchronization if complex operations are performed.

The choice between these shared state mechanisms involves a trade-off. Manager objects offer the convenience of sharing rich Python data structures like lists and dictionaries with relative ease, as the underlying proxies and server process handle much of the synchronization complexity. However, this convenience comes at the cost of performance. Every operation on a managed object through its proxy involves inter-process communication with the manager's server process, and potentially serialization if method arguments or return values are complex. This indirection makes Manager objects inherently slower than direct shared memory access.  

On the other hand, Value and Array provide direct access to blocks of shared memory, which can be extremely fast for simple data types like integers or floats, akin to C-level pointer manipulation. This avoids the serialization overhead for data access. The "cost" here is reduced flexibility—they are restricted to basic C data types—and the need for more explicit and careful synchronization if the built-in lock is insufficient or if operations are not inherently atomic. For instance, incrementing a shared counter might require an explicit lock to ensure atomicity if the built-in lock is disabled or if multiple fields of a shared structure are updated together. Thus, for high-performance shared flags, counters, or small arrays of raw data, Value and Array are often superior. For sharing more complex Python collections where ease of use is paramount and performance is less critical, Manager objects are a suitable choice. It is often advised to minimize shared state where possible and prefer message passing via Queues or Pipes, as shared state introduces complexities related to synchronization and potential performance bottlenecks.  

The Mechanics of Data Serialization: pickle and the dill Alternative

The multiprocessing module, by default, relies heavily on Python's pickle module for data serialization. Serialization is the process of converting a Python object into a byte stream that can be stored or transmitted, and deserialization is the reverse process of reconstructing the object from the byte stream. This is essential when passing arguments to functions in different processes, sending items through Queues or Pipes, or returning results from worker processes.  

However, pickle has limitations. It cannot serialize certain types of Python objects, including lambda functions, functions defined interactively in the interpreter, nested functions (closures), and some generators or dynamically created classes. When multiprocessing attempts to transfer such an unpicklable object, a PicklingError will occur, halting the operation.  

For scenarios where pickle's capabilities are insufficient, the dill library offers a more robust alternative. dill can serialize a much wider range of Python objects, including those that pickle cannot handle. The multiprocess package, which is a fork of the standard multiprocessing module, integrates dill as its default serializer, thereby providing enhanced serialization capabilities out-of-the-box. This can be particularly useful in scientific computing or complex applications where such objects are common.  

Data serialization is not just a functional prerequisite for multiprocessing but can also be a significant performance consideration and a source of errors. The process of pickling and unpickling large or intricate data structures can be computationally expensive and consume considerable time. If the data being transferred between processes is substantial, this serialization overhead can become a bottleneck, potentially diminishing or even negating the performance gains achieved through parallelism.  

Furthermore, the compatibility issues stemming from pickle's limitations can force developers to refactor their code to avoid unpicklable objects or to adopt alternative serialization libraries like dill. This adds a layer of complexity to development. While less common in typical self-contained multiprocessing applications, there is also a security consideration: unpickling data received from an untrusted source via a Connection object (from a Pipe) can pose a security risk if the pickled data is maliciously crafted.  

Therefore, developers must be mindful of the nature and volume of data being passed between processes. Effective strategies include:

    Transferring only the necessary data.
    Employing shared memory mechanisms (Value, Array) for large numerical datasets to bypass serialization for data access.
    Considering the multiprocess package (which uses dill) if pickle proves inadequate for the types of objects being used.
    Understanding that the choice of IPC mechanism can also influence serialization overhead; for instance, Manager proxies might lead to more frequent serialization events for method calls compared to a direct Queue.put() of a simple, already-pickled object.

The following table summarizes the key IPC mechanisms available in multiprocessing:

Table 2: Overview of Inter-Process Communication (IPC) Mechanisms
Mechanism	Description	Typical Use Case	Serialization Overhead	Synchronization	Complexity/Overhead
multiprocessing.Queue	Process-safe FIFO queue for message passing.	Task distribution, result collection, general messaging.	Pickling (e.g., pickle/dill)	Built-in	Moderate
multiprocessing.Pipe()	Returns a pair of connection objects for two-way communication.	Direct communication between two specific processes.	Pickling (e.g., pickle/dill)	User-managed (if needed beyond basic send/recv)	Low to Moderate
multiprocessing.Manager	Server process hosts shared Python objects (lists, dicts) via proxies.	Sharing complex Python objects (e.g., lists, dicts).	Pickling for proxy method args/returns	Built-in for managed types	High (due to server process and proxies)
multiprocessing.Value	Shared memory for a single C-type variable.	Sharing simple numeric values, flags.	Minimal (for setup)	Optional built-in lock, or user-managed	Low (for basic types)
multiprocessing.Array	Shared memory for an array of C-type variables.	Sharing arrays of simple numeric data.	Minimal (for setup)	Optional built-in lock, or user-managed	Low (for basic types)
5. Ensuring Order: Synchronization Primitives

When multiple processes access and modify shared resources concurrently, there's a risk of race conditions—situations where the final state of the shared resource depends on the unpredictable order of execution of operations by different processes. This can lead to corrupted data or incorrect program behavior. Python's multiprocessing module provides synchronization primitives, similar to those in the threading module, to manage access to shared resources.
Preventing Race Conditions with Lock

The most fundamental synchronization primitive is the Lock. A multiprocessing.Lock object can be used to ensure that only one process at a time can execute a critical section of code or access a particular shared resource.
A process attempts to acquire the lock using lock.acquire(). If the lock is already held by another process, acquire() will block until the lock is released. Once a process has acquired the lock, it can safely access the shared resource. After it's done, it must release the lock using lock.release(), allowing another waiting process to acquire it.
A common and safer pattern is to use the lock as a context manager with the with statement:  

Python

# import multiprocessing
# lock = multiprocessing.Lock()
# shared_resource = multiprocessing.Value('i', 0)

# def critical_section_worker(lock, shared_resource):
#     with lock:
#         # Access and modify shared_resource safely
#         shared_resource.value += 1

The with lock: statement automatically calls lock.acquire() at the beginning of the block and lock.release() at the end, even if errors occur within the block. This is crucial when using shared memory objects like Value or Array, or when interacting with Manager objects if the operations involved are not inherently atomic or if multiple operations need to be grouped together to form a single atomic transaction.
Brief Overview of Other Primitives

While Lock is often sufficient, multiprocessing provides other synchronization tools for more complex scenarios:

    RLock (Re-entrant Lock): An RLock allows the same process to acquire it multiple times without deadlocking. The lock must be released an equal number of times it was acquired before another process can acquire it. This is useful if a function that holds a lock needs to call another function that also needs to acquire the same lock.   

Semaphore / BoundedSemaphore: A semaphore manages an internal counter which is decremented by each acquire() call and incremented by each release() call. It blocks acquire() if the counter is zero. This allows a limited number of processes to access a resource or a pool of resources concurrently. A BoundedSemaphore additionally ensures that release() is not called more times than acquire(), raising a ValueError if this occurs.  
Condition: Condition variables allow one or more processes to wait until they are notified by another process that some condition has become true. They are typically associated with a Lock (or RLock) and have wait(), notify(), and notify_all() methods. These are useful for implementing more complex producer-consumer patterns or other coordination schemes.  

Synchronization primitives like Lock are indispensable for ensuring data integrity when processes share mutable resources. However, their use is a double-edged sword. When a process acquires a lock, any other process attempting to acquire the same lock must block and wait. This effectively serializes access to the code or resource protected by the lock. If these critical sections are large, frequently accessed, or if locks are held for extended periods, they can become significant performance bottlenecks, thereby reducing the overall parallelism achieved by the application. In worst-case scenarios, overuse or improper placement of locks can even lead to performance that is inferior to a purely sequential program.  

Furthermore, careless use of multiple locks can lead to deadlocks, a situation where two or more processes are blocked indefinitely, each waiting for a resource held by another process in the group. For example, if Process A acquires Lock 1 and then tries to acquire Lock 2, while Process B acquires Lock 2 and then tries to acquire Lock 1, both may block forever.  

Consequently, while synchronization is crucial for correctness in the presence of shared state, it must be applied judiciously. Critical sections protected by locks should be kept as small and efficient as possible. The most robust approach to designing multiprocessing applications often involves minimizing or eliminating shared mutable state altogether. This reduces the need for complex locking schemes and their associated risks, often favoring message-passing paradigms (using Queues or Pipes) where data is copied between processes rather than shared directly.  

6. Robustness in Parallel: Advanced Error Handling Strategies

Building robust multiprocessing applications requires careful attention to error handling. An unhandled exception in a worker process can lead to unexpected behavior, data loss, or a hung application if not managed correctly.
Capturing Exceptions within Worker Processes

An unhandled exception raised within a worker function (e.g., a function executed by Pool.map or as the target of a Process object) will typically cause that worker process to terminate. The main process might not receive a clear error message immediately, or it might hang indefinitely if it's waiting for a result from the failed worker.

To prevent abrupt termination and to allow for graceful error management, it's essential to include try...except blocks within the worker functions themselves. This allows the worker to catch expected exceptions, potentially log error details, and then decide how to proceed—perhaps by returning a special error indicator or a partial result.  

For example, a worker function could be structured as:
Python

# def worker_task_with_error_handling(data):
#     try:
#         #... perform computation...
#         if some_error_condition(data):
#             raise ValueError("Specific error occurred in worker")
#         result = process(data)
#         return {"status": "success", "result": result}
#     except ValueError as ve:
#         # Log the error, e.g., import logging; logging.error(f"ValueError in worker: {ve}")
#         return {"status": "error", "message": str(ve), "data": data}
#     except Exception as e:
#         # Catch any other unexpected errors
#         # Log the error
#         return {"status": "critical_error", "message": str(e), "data": data}

This approach allows the worker to package error information and return it as a normal result, which the main process can then inspect.
Communicating Errors from Workers to the Main Process

Several strategies exist for communicating error information from worker processes back to the main process:

    Built-in Pool Mechanisms: When using Pool.map() or Pool.starmap(), if a worker function raises an exception, that exception is typically caught by the Pool machinery. When the main process attempts to retrieve the results (e.g., by iterating over the list returned by map), the exception will be re-raised in the main process at the point corresponding to the failed task. Similarly, if Pool.apply_async() is used, calling get() on the returned AsyncResult object will re-raise any exception that occurred in the worker. This is often a convenient and straightforward way to handle errors.   

Using error_callback with Pool.apply_async: The apply_async method accepts an error_callback argument. If the target function in the worker raises an exception, this callback function will be executed in the main process, receiving the exception instance as an argument. This allows for centralized error handling for asynchronous tasks without needing to try...except around every get() call.  
Custom Error Reporting with Queue: For more fine-grained control, or when not using Pool's built-in mechanisms (e.g., with manually managed Process objects), a dedicated multiprocessing.Queue can be used for error reporting. Worker processes can catch exceptions and, instead of letting the Pool handle them, put detailed error information (e.g., an error object, a custom dictionary, or a tuple indicating failure type and details) onto this shared error queue. The main process can then periodically check this queue to retrieve and process error reports. This allows for non-blocking error collection and more complex error recovery logic.  

The failure to properly capture and communicate errors from worker processes can lead to a pernicious problem often termed "silent failures." In such scenarios, the main process might continue its execution, oblivious to the fact that one or more subsidiary tasks have failed, potentially leading to incorrect overall results or corrupted data. Alternatively, if the main process is waiting for results from workers that have crashed (e.g., via Process.join() or AsyncResult.get() without a timeout), it might hang indefinitely.

A worker process is an independent operating system entity. If it terminates unexpectedly due to an unhandled exception, the main process might only become aware of this implicitly—for example, a Pool might silently replace a dead worker, or a join() call on a Process object might return unexpectedly. While Pool.map() and AsyncResult.get() are generally effective at propagating exceptions back to the caller, this relies on the main process actively attempting to retrieve the result or iterate through the collective results. If tasks are launched in a "fire-and-forget" manner, or if there's a flaw in the result collection logic, errors can easily be overlooked.

Therefore, it is insufficient for worker processes merely to avoid crashing; they must be designed to actively report any failures encountered. The main process, in turn, must implement a robust strategy to check for and handle these reported errors. This might involve the worker returning a special sentinel value or a custom exception object that the main process can identify, or using a dedicated error communication channel like an error Queue or the error_callback mechanism. The issue of a "Pool is still running" error, which can occur if child processes do not terminate cleanly, can also be exacerbated by unhandled exceptions within workers that cause them to hang or exit improperly. Proactive error propagation and handling are key to building resilient multiprocessing systems.  

7. Best Practices for Effective Python Multiprocessing

Achieving efficient and reliable parallel processing with Python's multiprocessing module involves adhering to several best practices that address common pitfalls and optimize performance.

    Designing for Parallelism: Minimizing Shared State: One of the most effective strategies is to design applications to minimize or, ideally, avoid shared mutable state between processes. When processes operate on independent data, the need for complex synchronization primitives (like locks) is reduced, which in turn simplifies the code and lessens the risk of deadlocks or performance bottlenecks caused by contention for shared resources. Prefer passing data to worker processes as arguments and receiving results as return values. If ongoing communication is necessary, message passing via Queues or Pipes is generally safer and easier to reason about than shared memory.   

Task Granularity: As previously discussed (Section 3), the amount of work done by each task (its granularity) is critical. Tasks should be substantial enough to offset the overhead of process creation and inter-process communication. If tasks are too small, the overhead can dominate; if too large and varied, load balancing can suffer. Profiling is often necessary to find the right balance.
Resource Management: Joining Processes and Context Managers:

    When creating processes explicitly using the Process class, it is imperative to call join() on each process object before the main program exits. This ensures that the main program waits for the child processes to complete their execution and release system resources. Failing to join processes can lead to "zombie" processes on POSIX systems.   

For multiprocessing.Pool objects, using them as context managers (with Pool(...) as p:) is highly recommended. This pattern automatically calls pool.close() (preventing new tasks from being submitted) and pool.join() (waiting for all worker processes to complete their current tasks and exit) when the with block is exited, even if exceptions occur.  

Serialization Awareness: Be conscious of the data being passed between processes (Section 4). The default serializer, pickle, has limitations and can incur performance costs for large or complex objects. Understand what can and cannot be pickled. If PicklingErrors are encountered or if serialization overhead is a concern, consider:

    Simplifying the data structures being transferred.
    Using shared memory (Value, Array) for large numerical data to avoid repeated serialization.
    Exploring the multiprocess library, which uses the more capable dill serializer.   

Graceful Shutdown: Design worker functions to terminate cleanly. Avoid relying on Process.terminate(), which forcibly kills a process and can lead to shared resources (like locks or data in queues) being left in an inconsistent or corrupted state. Instead, use mechanisms like Events or sentinel values in queues to signal workers to shut down gracefully.  
Logging in Multiprocessing Applications: Standard Python logging can be problematic in multiprocessing applications because multiple processes might try to write to the same log file or console simultaneously, leading to garbled output. A robust solution is to use a logging.handlers.QueueHandler. Each worker process sends its log records to a multiprocessing.Queue. A separate thread or process in the main application (or a dedicated logging process) can then listen to this queue and handle the log records sequentially, writing them to the desired destination (file, console, etc.).  
Testing and Debugging: Parallel programs can be more challenging to debug than sequential ones due to non-deterministic behavior and the difficulty of inspecting state across multiple processes. Test individual components (like worker functions) in isolation first. Utilize extensive logging to trace execution flow and variable states.
Considering the multiprocess Library for Enhanced Serialization: The Python standard library aims for stability and broad applicability, which sometimes means not including features that might be more specialized or have more complex implications, such as an advanced object serializer. pickle, the default serializer in multiprocessing, is a case in point; its inability to serialize certain common Python constructs like lambdas, closures, or dynamically defined functions and classes is a known practical limitation for developers working with more intricate codebases. The open-source community often addresses such gaps. The dill library was developed to be a more comprehensive serializer, and subsequently, the multiprocess package was forked from the standard library specifically to integrate dill by default. This provides users with a readily available solution if the standard library's serialization capabilities prove insufficient. While this empowers users to tackle more complex parallelization tasks, it also necessitates awareness of these third-party alternatives and the management of additional dependencies. This ecosystem response also underscores that even limitations in core libraries can be effectively addressed by community-driven solutions.  

8. Conclusion: Mastering 24-Process Parallelism

This report has detailed the methodology for leveraging Python's multiprocessing module to achieve true parallelism, specifically targeting the execution of 24 concurrent processes. The core distinction between processes and threads, primarily dictated by Python's Global Interpreter Lock (GIL), establishes multiprocessing as the preferred solution for CPU-bound tasks. The provided example program demonstrates the practical application of multiprocessing.Pool to distribute work across 24 worker processes, manage task execution, and collect results, all while adhering to essential practices like the if __name__ == '__main__': guard.

Key techniques explored include various methods for inter-process communication (IPC) such as Queues, Pipes, and shared state mechanisms like Manager objects, Value, and Array. The critical role of data serialization, its potential bottlenecks, and the dill alternative via the multiprocess package were also examined. Furthermore, the necessity of synchronization primitives like Locks for preventing race conditions in shared-state scenarios, alongside robust error handling strategies within and between processes, has been emphasized.

Effective parallel programming in Python extends beyond simply launching multiple processes. It requires careful consideration of task granularity to balance computational work against overhead, diligent resource management, and an awareness of the nuances of data sharing and synchronization.

For further exploration and optimization, developers should consider:

    Profiling: Using tools like cProfile and snakeviz to identify performance bottlenecks in both the main process and worker functions. This can help optimize task granularity and pinpoint overly expensive operations or serialization costs.
    concurrent.futures.ProcessPoolExecutor: This class, found in the concurrent.futures module, provides a higher-level abstraction for managing a pool of processes and shares many similarities with multiprocessing.Pool. It offers a more modern API and can be a good alternative, especially for new projects.
    Asynchronous Programming with asyncio: For applications that are predominantly I/O-bound rather than CPU-bound, Python's asyncio framework, in conjunction with aiohttp for network requests or aiofiles for file operations, can offer highly efficient concurrency with lower overhead than multiprocessing or even threading in many cases.

By understanding the principles and practices outlined in this report, developers can effectively harness the power of multi-core processors with Python, building scalable and performant applications capable of handling demanding computational workloads.
geeksforgeeks.org
Multiprocessing in Python | Set 1 (Introduction) | GeeksforGeeks
Opens in a new window
paulnorvig.com
Tutorial: Parallel Programming with multiprocessing in Python (2024)
Opens in a new window
geeksforgeeks.org
www.geeksforgeeks.org
Opens in a new window
mimo.org
Python Multiprocessing: Syntax, Usage, and Examples - Mimo
Opens in a new window
builtin.com
Python Multithreading vs. Multiprocessing Explained | Built In
Opens in a new window
multiprocess.readthedocs.io
multiprocess package documentation — multiprocess 0.70.19.dev0 documentation
Opens in a new window
docs.python.org
multiprocessing — Process-based parallelism — Python 3.13.3 ...
Opens in a new window
blog.heycoach.in
Handling Exceptions In Multi-processing In Python - HeyCoach | Blogs
Opens in a new window
labex.io
How to manage 'Pool is still running' error - LabEx
Opens in a new window
matthewrocklin.com
Parallelism and Serialization - Matthew Rocklin