i am fetching data using serial port in my python program. the data contain 30000 data points. which i have to save in CSV files but the data is not saving in csv files.
from configuration.config import COM_PORT, BAUD_RATE
import serial
import threading
import time
import logging
import os
import csv
import requests
buffer1 = []
buffer2 = []
active_buffer = buffer1
lock = threading.Lock()
text_file_name = ‘data.txt’
csv_file_prefix = ‘data’
csv_file_counter = 1
csv_file_name = f”{csv_file_prefix}_{csv_file_counter}.csv”
stop_event = threading.Event()
highest_srno = 29999
def get_new_csv_filename():
global csv_file_counter
filename = f”{csv_file_prefix}_{csv_file_counter}.csv”
csv_file_counter += 1
return filename
def read_data():
global active_buffer, highest_srno, csv_file_name
try:
ser = serial.Serial(COM_PORT, BAUD_RATE)
print(f”Connected to the sensor with the above credentials”)
if ser.in_waiting > 0: # Check if there is data available to read
data = ser.readline().decode(‘utf-8’).strip()
try:
SerialNo, x, y, z = map(int, data.split(‘,’))
print(f”SRNO:{SerialNo}, X:{x}, Y:{y}, Z:{z}”)
# # This is the code to send the data to API
# payload = {
# “sr_no”: SerialNo,
# “x”: x,
# “y”: y,
# “z”: z
# }
# try:
# response = requests.post(API_endpoints, json=payload, timeout=10)
# if response.status_code == 200:
# print(“Data sent to API”)
# else:
# print(f”Failed to send data to API {response.status_code} – {response.text}”)
# except requests.exceptions.RequestException as e:
# print(f”Request failed: {e}”)
with lock:
if SerialNo > highest_srno:
if highest_srno == 29999:
csv_file_name = get_new_csv_filename()
highest_srno = 0
else:
highest_srno = SerialNo
active_buffer.append(f”SRNO:{SerialNo},X:{x},Y:{y},Z:{z}”)
except ValueError:
print(f”Error: Data format incorrect, skipping this line”)
except Exception as e:
logging.error(f”Error: {e}”)
finally:
if ser.is_open:
ser.close()
print(“Serial connection closed.”)
def write_data_to_file():
global active_buffer, buffer1, buffer2, csv_file_name
while not stop_event.is_set():
time.sleep(4) # Periodically write data every 4 seconds
with lock:
buffer_to_write = active_buffer
if active_buffer == buffer1:
active_buffer = buffer2
else:
active_buffer = buffer1
try:
# Write to text file
with open(text_file_name, ‘a’) as f:
for line in buffer_to_write:
f.write(line + ‘n’)
# Write to CSV file
with open(csv_file_name, ‘a’, newline=”) as csvfile:
csv_writer = csv.writer(csvfile)
for line in buffer_to_write:
parts = line.split(‘,’)
serial_no = parts[0].split(‘:’)[1]
x = parts[1].split(‘:’)[1]
y = parts[2].split(‘:’)[1]
z = parts[3].split(‘:’)[1]
csv_writer.writerow([serial_no, x, y, z])
buffer_to_write.clear()
except IOError as e:
logging.error(f”File error: {e}”)
if name == “main“:
logging.basicConfig(level=logging.ERROR, format=’%(asctime)s – %(levelname)s – %(message)s’)
write_thread.daemon = True
write_thread.start()
try:
read_data()
except KeyboardInterrupt:
print(“Stopping…”)
stop_event.set()
write_thread.join()
print(“Program terminated.”)
