NumPy, Pandas & Matplotlib

pip install numpy pandas matplotlib seaborn openpyxl

1import numpy as np
2
3# From a Python list
4a = np.array([10, 20, 30, 40, 50])
5print(a)          # [10 20 30 40 50]
6print(a.dtype)    # int64
7print(a.shape)    # (5,)
8
9# Specify data type
10b = np.array([1.5, 2.5, 3.5], dtype=np.float32)
11
12# Array of zeros / ones / uninitialized
13zeros  = np.zeros((3, 4))       # 3 rows, 4 columns of 0.0
14ones   = np.ones((2, 3))        # 2 rows, 3 columns of 1.0
15empty  = np.empty((2, 2))       # Uninitialized (random garbage values)
16eye    = np.eye(3)              # 3x3 Identity matrix
17
18# Sequences
19seq    = np.arange(0, 10, 2)   # [0 2 4 6 8]  (start, stop, step)
20lin    = np.linspace(0, 1, 5)  # [0.   0.25  0.5   0.75  1.  ]  (start, stop, count)
21
22# Random arrays
23rng = np.random.default_rng(seed=42)
24rand_f = rng.random((3, 3))          # Uniform [0, 1)
25rand_n = rng.normal(0.0, 1.0, (100,))  # Standard normal distribution

1import numpy as np
2
3arr = np.array([10, 20, 30, 40, 50, 60])
4
5print(arr[0])     # 10  (first element)
6print(arr[-1])    # 60  (last element)
7print(arr[1:4])   # [20 30 40]  (slice: index 1 to 3 inclusive)
8print(arr[::2])   # [10 30 50]  (every 2nd element)
9
10# 2D indexing
11matrix = np.array([
12    [1, 2, 3],
13    [4, 5, 6],
14    [7, 8, 9]
15])
16
17print(matrix[0, :])    # [1 2 3]  — first row
18print(matrix[:, 1])    # [2 5 8]  — second column
19print(matrix[1, 2])    # 6        — row 1, col 2
20print(matrix[:2, :2])  # Top-left 2x2 block
21
22# Boolean (conditional) indexing
23sensor = np.array([21.5, 35.2, 18.9, 42.0, 28.1])
24hot    = sensor[sensor > 30]         # [35.2 42. ]
25print(hot)

1import numpy as np
2
3temps_c = np.array([0.0, 20.0, 37.0, 100.0])
4
5# Broadcasting: operation applies to every element
6temps_f = (temps_c * 1.8) + 32
7print(temps_f)   # [32.  68.  98.6  212. ]
8
9a = np.array([1, 2, 3])
10b = np.array([4, 5, 6])
11
12print(a + b)     # [ 5  7  9]
13print(a * b)     # [ 4 10 18]
14print(np.dot(a, b))  # 32   (dot product: 1*4 + 2*5 + 3*6)
15
16# Aggregation
17readings = np.array([22.1, 23.5, 21.8, 24.9, 22.7])
18print(readings.mean())    # 23.0
19print(readings.std())     # 1.07
20print(readings.min())     # 21.8
21print(readings.max())     # 24.9
22print(readings.sum())     # 115.0
23print(np.median(readings))  # 22.7
24print(np.argmax(readings))  # 3  (index of max value)

1import numpy as np
2
3a = np.arange(12)          # [ 0  1  2  3  4  5  6  7  8  9 10 11]
4b = a.reshape(3, 4)        # Shape (3, 4)
5c = a.reshape(2, 2, 3)     # Shape (2, 2, 3)
6
7# Flatten back to 1D
8flat = b.flatten()
9
10# Stack arrays together
11x = np.array([1, 2, 3])
12y = np.array([4, 5, 6])
13
14h_stack = np.hstack([x, y])         # [1 2 3 4 5 6]  — side by side
15v_stack = np.vstack([x, y])         # 2D: two rows
16
17# Transpose a matrix
18m = np.array([[1, 2, 3], [4, 5, 6]])
19print(m.T)   # Shape (3, 2)

1import numpy as np
2
3A = np.array([[2, 1], [5, 3]])
4B = np.array([[1, 2], [3, 4]])
5
6print(np.dot(A, B))         # Matrix multiplication
7print(np.linalg.inv(A))     # Inverse of A
8print(np.linalg.det(A))     # Determinant
9eigenvalues, eigenvectors = np.linalg.eig(A)
10print(eigenvalues)

1import pandas as pd
2
3# From a Python dictionary
4data = {
5    "name":        ["Alice", "Bob", "Charlie", "Diana"],
6    "age":         [25, 31, 28, 35],
7    "temperature": [36.6, 37.1, 36.9, 38.2],
8    "city":        ["BLR", "HYD", "BLR", "DEL"],
9}
10
11df = pd.DataFrame(data)
12print(df)

      name  age  temperature city
0    Alice   25         36.6  BLR
1      Bob   31         37.1  HYD
2  Charlie   28         36.9  BLR
3    Diana   35         38.2  DEL

1print(df.shape)         # (4, 4) — 4 rows, 4 columns
2print(df.columns)       # Index(['name', 'age', 'temperature', 'city'], ...)
3print(df.dtypes)        # Data type of each column
4print(df.info())        # Column types + non-null counts
5print(df.describe())    # Statistics: count, mean, std, min, max, quartiles
6print(df.head(2))       # First 2 rows
7print(df.tail(2))       # Last 2 rows
8print(df.sample(2))     # 2 random rows

1# Single column → Series
2age_series = df["age"]
3
4# Multiple columns → DataFrame
5subset = df[["name", "temperature"]]
6
7# Single row by index label (iloc = position-based, loc = label-based)
8row0   = df.iloc[0]          # First row
9row_by = df.loc[0]           # Row with index label 0
10
11# Slicing rows
12first_two = df.iloc[:2]
13
14# Boolean filtering
15hot_patients = df[df["temperature"] > 37]
16blr_patients = df[df["city"] == "BLR"]
17
18# Multiple conditions
19blr_hot = df[(df["city"] == "BLR") & (df["temperature"] > 36.5)]
20
21# isin() for multiple values
22df[df["city"].isin(["BLR", "DEL"])]

1# New column from calculation
2df["temp_f"] = (df["temperature"] * 1.8) + 32
3
4# Conditional column using np.where
5import numpy as np
6df["status"] = np.where(df["temperature"] > 37, "Fever", "Normal")
7
8# Apply a custom function to each row
9df["name_upper"] = df["name"].apply(lambda x: x.upper())
10
11# Rename columns
12df = df.rename(columns={"temperature": "temp_c"})
13
14# Drop a column
15df = df.drop(columns=["name_upper"])

1import pandas as pd
2import numpy as np
3
4# Create a DataFrame with missing values
5df = pd.DataFrame({
6    "sensor_id": [1, 2, 3, 4, 5],
7    "temp":      [22.5, np.nan, 23.1, np.nan, 24.0],
8    "humidity":  [55.0, 60.0, np.nan, 65.0, 70.0],
9})
10
11print(df.isna().sum())           # Count missing per column
12print(df.isna().any(axis=1))     # True for rows with ANY missing value
13
14# Strategy 1: Drop rows with any missing value
15df_dropped = df.dropna()
16
17# Strategy 2: Drop only if ALL values in a row are missing
18df_drop_all = df.dropna(how="all")
19
20# Strategy 3: Fill missing with a constant
21df["temp"] = df["temp"].fillna(0)
22
23# Strategy 4: Fill with column mean (most common for sensor data)
24df["humidity"] = df["humidity"].fillna(df["humidity"].mean())
25
26# Strategy 5: Forward fill (use last known value)
27df["temp"] = df["temp"].ffill()

1df = pd.DataFrame({
2    "device":  ["A", "B", "A", "B", "A"],
3    "reading": [22.1, 30.5, 21.8, 31.2, 23.0],
4    "hour":    [8, 8, 9, 9, 10],
5})
6
7# Average reading per device
8print(df.groupby("device")["reading"].mean())
9
10# Multiple aggregations at once
11stats = df.groupby("device")["reading"].agg(["mean", "std", "min", "max"])
12print(stats)
13
14# Group by multiple columns
15pivot = df.groupby(["device", "hour"])["reading"].mean().reset_index()
16print(pivot)
17
18# Value counts — how many rows per category
19print(df["device"].value_counts())

1# Sort by a single column (descending)
2df_sorted = df.sort_values("reading", ascending=False)
3
4# Sort by multiple columns
5df_sorted2 = df.sort_values(["device", "hour"])
6
7# Rank values
8df["rank"] = df["reading"].rank(ascending=False)

1import pandas as pd
2
3# Read
4df = pd.read_csv("sensor_data.csv")
5
6# Read with options
7df = pd.read_csv(
8    "sensor_data.csv",
9    sep=",",               # Separator (default: comma)
10    header=0,              # Row to use as column names
11    index_col="timestamp", # Set a column as the index
12    parse_dates=["timestamp"],  # Parse this column as datetime
13    usecols=["timestamp", "temperature", "humidity"],  # Load only these columns
14    nrows=1000,            # Load only first 1000 rows (great for large files)
15)
16
17# Write
18df.to_csv("output.csv", index=False)  # index=False avoids writing row numbers

1import pandas as pd
2import json
3
4# --- Approach 1: pandas read_json ---
5# Works when the file is a list of records [{...}, {...}] or nested dict
6df = pd.read_json("records.json")
7df.to_json("output.json", orient="records", indent=2)
8
9# --- Approach 2: Python json module (for complex nested JSON) ---
10with open("nested_data.json", "r") as f:
11    raw = json.load(f)
12
13# Example: extract a nested field from an API response
14# raw = {"status": 200, "data": [{"id": 1, "temp": 22.5}, ...]}
15records = raw["data"]
16df = pd.DataFrame(records)
17print(df.head())
18
19# --- Approach 3: JSON string from an MQTT message ---
20payload_str = '{"device_id": "ESP32_01", "temperature": 28.5, "humidity": 64}'
21payload = json.loads(payload_str)        # String → dict
22print(payload["temperature"])            # 28.5
23json_string = json.dumps(payload)        # dict → string

1import pandas as pd
2
3# Read first sheet (requires openpyxl)
4df = pd.read_excel("report.xlsx")
5
6# Read a specific sheet by name
7df_sheet2 = pd.read_excel("report.xlsx", sheet_name="Sales_2024")
8
9# Read multiple sheets at once — returns a dict of DataFrames
10all_sheets = pd.read_excel("report.xlsx", sheet_name=None)
11for sheet_name, sheet_df in all_sheets.items():
12    print(f"Sheet: {sheet_name}, Rows: {len(sheet_df)}")
13
14# Write to Excel (single sheet)
15df.to_excel("output.xlsx", sheet_name="Results", index=False)
16
17# Write multiple DataFrames to different sheets
18with pd.ExcelWriter("multi_sheet.xlsx", engine="openpyxl") as writer:
19    df.to_excel(writer, sheet_name="Summary", index=False)
20    df_sheet2.to_excel(writer, sheet_name="Details", index=False)

1import matplotlib.pyplot as plt
2import numpy as np
3
4time = np.arange(0, 24)  # 0 to 23 hours
5temp = 25 + 5 * np.sin(time * np.pi / 12)  # Simulated daily temperature curve
6
7plt.figure(figsize=(10, 4))
8plt.plot(time, temp, color="tomato", linewidth=2, label="Temperature (°C)")
9plt.title("Daily Temperature Curve")
10plt.xlabel("Hour of Day")
11plt.ylabel("Temperature (°C)")
12plt.xticks(range(0, 24, 2))
13plt.legend()
14plt.grid(True, linestyle="--", alpha=0.5)
15plt.tight_layout()
16plt.show()

1import matplotlib.pyplot as plt
2
3devices  = ["Sensor A", "Sensor B", "Sensor C", "Sensor D"]
4avg_temp = [22.5, 31.2, 27.8, 19.4]
5colors   = ["steelblue", "tomato", "green", "orange"]
6
7plt.figure(figsize=(8, 5))
8bars = plt.bar(devices, avg_temp, color=colors, edgecolor="black", width=0.5)
9plt.title("Average Temperature by Sensor")
10plt.ylabel("Avg Temperature (°C)")
11
12# Add value labels on top of each bar
13for bar in bars:
14    plt.text(
15        bar.get_x() + bar.get_width() / 2,
16        bar.get_height() + 0.3,
17        f"{bar.get_height():.1f}°C",
18        ha="center", fontsize=11
19    )
20
21plt.tight_layout()
22plt.show()

1import matplotlib.pyplot as plt
2import numpy as np
3
4data = np.random.normal(loc=37.0, scale=0.5, size=1000)
5
6plt.figure(figsize=(8, 4))
7plt.hist(data, bins=30, color="steelblue", edgecolor="white", alpha=0.8)
8plt.axvline(data.mean(), color="red", linestyle="--", label=f"Mean: {data.mean():.2f}")
9plt.title("Distribution of Body Temperatures")
10plt.xlabel("Temperature (°C)")
11plt.ylabel("Frequency")
12plt.legend()
13plt.show()

1import matplotlib.pyplot as plt
2import numpy as np
3
4np.random.seed(42)
5humidity    = np.random.uniform(40, 90, 200)
6temperature = 15 + 0.4 * humidity + np.random.normal(0, 3, 200)
7
8plt.figure(figsize=(8, 5))
9plt.scatter(humidity, temperature, alpha=0.5, color="teal", edgecolors="none", s=30)
10plt.title("Temperature vs. Humidity")
11plt.xlabel("Humidity (%)")
12plt.ylabel("Temperature (°C)")
13plt.grid(True, linestyle="--", alpha=0.4)
14plt.show()

1import matplotlib.pyplot as plt
2import numpy as np
3
4x  = np.linspace(0, 2 * np.pi, 100)
5
6fig, axes = plt.subplots(2, 2, figsize=(10, 8))
7fig.suptitle("Signal Analysis Dashboard", fontsize=14)
8
9axes[0, 0].plot(x, np.sin(x), color="steelblue")
10axes[0, 0].set_title("Sine Wave")
11
12axes[0, 1].plot(x, np.cos(x), color="tomato")
13axes[0, 1].set_title("Cosine Wave")
14
15axes[1, 0].plot(x, np.sin(x) + np.cos(x), color="green")
16axes[1, 0].set_title("Sin + Cos")
17
18axes[1, 1].plot(x, np.sin(2 * x), color="purple")
19axes[1, 1].set_title("Sin(2x)")
20
21plt.tight_layout()
22plt.show()

1import seaborn as sns
2import pandas as pd
3import matplotlib.pyplot as plt
4import numpy as np
5
6# Create synthetic sensor DataFrame
7df = pd.DataFrame(np.random.rand(200, 4), columns=["Temp", "Humidity", "Pressure", "CO2"])
8
9corr = df.corr()
10
11plt.figure(figsize=(6, 5))
12sns.heatmap(
13    corr, annot=True, fmt=".2f",
14    cmap="coolwarm", vmin=-1, vmax=1,
15    linewidths=0.5
16)
17plt.title("Feature Correlation Matrix")
18plt.show()

1import pandas as pd
2import matplotlib.pyplot as plt
3
4# 1. Load CSV
5df = pd.read_csv("sensor_log.csv", parse_dates=["timestamp"])
6
7# 2. Inspect
8print(df.shape)
9print(df.isna().sum())
10
11# 3. Clean
12df = df.dropna(subset=["temperature", "humidity"])
13df = df[df["temperature"].between(10, 50)]  # Remove physically impossible readings
14
15# 4. Feature engineering
16df["hour"]       = df["timestamp"].dt.hour
17df["temp_f"]     = (df["temperature"] * 1.8) + 32
18df["heat_index"] = df["temperature"] + 0.33 * df["humidity"] - 4
19
20# 5. Aggregate
21hourly_avg = df.groupby("hour")["temperature"].mean().reset_index()
22
23# 6. Export summary
24hourly_avg.to_csv("hourly_summary.csv", index=False)
25hourly_avg.to_excel("hourly_summary.xlsx", index=False)
26hourly_avg.to_json("hourly_summary.json", orient="records", indent=2)
27
28# 7. Plot
29plt.figure(figsize=(10, 4))
30plt.plot(hourly_avg["hour"], hourly_avg["temperature"], marker="o", color="steelblue")
31plt.title("Average Temperature by Hour of Day")
32plt.xlabel("Hour")
33plt.ylabel("Temperature (°C)")
34plt.xticks(range(24))
35plt.grid(True, linestyle="--", alpha=0.4)
36plt.tight_layout()
37plt.show()