Advanced Monitoring Patterns: Building Observable Applications in 2025

Introduction

Monitoring continues to be a critical aspect of modern applications in 2025, with new patterns and practices emerging to handle the increasing complexity of distributed systems. This comprehensive guide explores advanced monitoring patterns and their practical applications.

Metrics Collection Patterns

1. Prometheus Integration

// metrics.service.ts
interface MetricsConfig {
  prefix: string;
  defaultLabels: Record<string, string>;
  pushGateway: string;
}

class MetricsService {
  private readonly registry: Registry;
  private readonly metrics: Map<string, Metric>;

  constructor(private readonly config: MetricsConfig) {
    this.registry = new Registry();
    this.metrics = new Map();
    this.initializeMetrics();
  }

  private initializeMetrics() {
    // Counter for total requests
    this.metrics.set(
      'http_requests_total',
      new Counter({
        name: `${this.config.prefix}_http_requests_total`,
        help: 'Total number of HTTP requests',
        labelNames: ['method', 'path', 'status']
      })
    );

    // Histogram for request duration
    this.metrics.set(
      'http_request_duration_seconds',
      new Histogram({
        name: `${this.config.prefix}_http_request_duration_seconds`,
        help: 'HTTP request duration in seconds',
        labelNames: ['method', 'path'],
        buckets: [0.1, 0.5, 1, 2, 5]
      })
    );

    // Gauge for active connections
    this.metrics.set(
      'active_connections',
      new Gauge({
        name: `${this.config.prefix}_active_connections`,
        help: 'Number of active connections',
        labelNames: ['type']
      })
    );

    // Register all metrics
    this.metrics.forEach(metric => this.registry.registerMetric(metric));
  }

  async recordRequest(
    method: string,
    path: string,
    status: number,
    duration: number
  ): Promise<void> {
    const labels = { method, path, status: status.toString() };
    
    this.metrics.get('http_requests_total').inc(labels);
    this.metrics.get('http_request_duration_seconds').observe(
      { method, path },
      duration
    );
  }

  setActiveConnections(type: string, count: number): void {
    this.metrics.get('active_connections').set({ type }, count);
  }

  async pushMetrics(): Promise<void> {
    try {
      await push({
        jobName: this.config.prefix,
        registry: this.registry,
        pushgateway: this.config.pushGateway
      });
    } catch (error) {
      console.error('Failed to push metrics:', error);
    }
  }
}

// Usage in middleware
@Injectable()
class MetricsMiddleware implements NestMiddleware {
  constructor(private readonly metricsService: MetricsService) {}

  async use(req: Request, res: Response, next: NextFunction) {
    const start = process.hrtime();

    res.on('finish', () => {
      const [seconds, nanoseconds] = process.hrtime(start);
      const duration = seconds + nanoseconds / 1e9;

      this.metricsService.recordRequest(
        req.method,
        req.path,
        res.statusCode,
        duration
      );
    });

    next();
  }
}

2. Custom Metrics

// custom-metrics.service.ts
interface CustomMetricsConfig {
  prefix: string;
  defaultLabels: Record<string, string>;
}

class CustomMetricsService {
  private readonly metrics: Map<string, Metric>;

  constructor(private readonly config: CustomMetricsConfig) {
    this.metrics = new Map();
    this.initializeMetrics();
  }

  private initializeMetrics() {
    // Business metrics
    this.metrics.set(
      'orders_total',
      new Counter({
        name: `${this.config.prefix}_orders_total`,
        help: 'Total number of orders',
        labelNames: ['status', 'payment_method']
      })
    );

    this.metrics.set(
      'order_value',
      new Histogram({
        name: `${this.config.prefix}_order_value`,
        help: 'Order value in currency',
        labelNames: ['currency'],
        buckets: [10, 50, 100, 500, 1000, 5000]
      })
    );

    // Performance metrics
    this.metrics.set(
      'cache_hits_total',
      new Counter({
        name: `${this.config.prefix}_cache_hits_total`,
        help: 'Total number of cache hits',
        labelNames: ['cache_type']
      })
    );

    this.metrics.set(
      'cache_misses_total',
      new Counter({
        name: `${this.config.prefix}_cache_misses_total`,
        help: 'Total number of cache misses',
        labelNames: ['cache_type']
      })
    );

    // Resource metrics
    this.metrics.set(
      'memory_usage_bytes',
      new Gauge({
        name: `${this.config.prefix}_memory_usage_bytes`,
        help: 'Memory usage in bytes',
        labelNames: ['type']
      })
    );

    this.metrics.set(
      'cpu_usage_percent',
      new Gauge({
        name: `${this.config.prefix}_cpu_usage_percent`,
        help: 'CPU usage percentage',
        labelNames: ['core']
      })
    );
  }

  recordOrder(
    status: string,
    paymentMethod: string,
    value: number,
    currency: string
  ): void {
    this.metrics.get('orders_total').inc({ status, payment_method: paymentMethod });
    this.metrics.get('order_value').observe({ currency }, value);
  }

  recordCacheHit(cacheType: string): void {
    this.metrics.get('cache_hits_total').inc({ cache_type: cacheType });
  }

  recordCacheMiss(cacheType: string): void {
    this.metrics.get('cache_misses_total').inc({ cache_type: cacheType });
  }

  setMemoryUsage(type: string, bytes: number): void {
    this.metrics.get('memory_usage_bytes').set({ type }, bytes);
  }

  setCPUUsage(core: string, percent: number): void {
    this.metrics.get('cpu_usage_percent').set({ core }, percent);
  }
}

// Usage in service
@Injectable()
class OrderService {
  constructor(
    private readonly metricsService: CustomMetricsService,
    private readonly cacheService: CacheService
  ) {}

  async createOrder(orderData: CreateOrderDto): Promise<Order> {
    // Try to get from cache
    const cachedOrder = await this.cacheService.get(`order:${orderData.id}`);
    if (cachedOrder) {
      this.metricsService.recordCacheHit('order');
      return cachedOrder;
    }
    this.metricsService.recordCacheMiss('order');

    // Create order
    const order = await this.orderRepository.create(orderData);
    
    // Record metrics
    this.metricsService.recordOrder(
      order.status,
      order.paymentMethod,
      order.value,
      order.currency
    );

    return order;
  }
}

Distributed Tracing Patterns

1. OpenTelemetry Integration

// tracing.service.ts
interface TracingConfig {
  serviceName: string;
  endpoint: string;
  sampler: Sampler;
}

class TracingService {
  private readonly tracer: Tracer;

  constructor(private readonly config: TracingConfig) {
    const provider = new NodeTracerProvider({
      sampler: this.config.sampler
    });

    provider.addSpanProcessor(
      new BatchSpanProcessor(
        new OTLPTraceExporter({
          url: this.config.endpoint
        })
      )
    );

    provider.register();
    this.tracer = provider.getTracer(this.config.serviceName);
  }

  startSpan(
    name: string,
    options: SpanOptions = {}
  ): Span {
    return this.tracer.startSpan(name, options);
  }

  async withSpan<T>(
    name: string,
    fn: (span: Span) => Promise<T>,
    options: SpanOptions = {}
  ): Promise<T> {
    const span = this.startSpan(name, options);
    
    try {
      const result = await fn(span);
      span.end();
      return result;
    } catch (error) {
      span.recordException(error);
      span.setStatus({ code: SpanStatusCode.ERROR });
      span.end();
      throw error;
    }
  }

  addEvent(span: Span, name: string, attributes?: Attributes): void {
    span.addEvent(name, attributes);
  }

  setAttributes(span: Span, attributes: Attributes): void {
    span.setAttributes(attributes);
  }
}

// Usage in service
@Injectable()
class UserService {
  constructor(
    private readonly tracingService: TracingService,
    private readonly userRepository: UserRepository
  ) {}

  async getUser(id: string): Promise<User> {
    return this.tracingService.withSpan(
      'get_user',
      async (span) => {
        span.setAttributes({ 'user.id': id });

        const user = await this.userRepository.findById(id);
        if (!user) {
          span.addEvent('user_not_found');
          throw new NotFoundError('User not found');
        }

        span.addEvent('user_found', {
          'user.email': user.email,
          'user.created_at': user.createdAt
        });

        return user;
      },
      {
        attributes: {
          'service.name': 'user-service',
          'operation.type': 'read'
        }
      }
    );
  }
}

2. Correlation IDs

// correlation.service.ts
interface CorrelationConfig {
  headerName: string;
  generateId: () => string;
}

class CorrelationService {
  constructor(private readonly config: CorrelationConfig) {}

  getCorrelationId(req: Request): string {
    const existingId = req.headers[this.config.headerName];
    if (existingId) {
      return Array.isArray(existingId) ? existingId[0] : existingId;
    }

    return this.config.generateId();
  }

  setCorrelationId(req: Request, res: Response): void {
    const correlationId = this.getCorrelationId(req);
    res.setHeader(this.config.headerName, correlationId);
  }
}

// Usage in middleware
@Injectable()
class CorrelationMiddleware implements NestMiddleware {
  constructor(private readonly correlationService: CorrelationService) {}

  use(req: Request, res: Response, next: NextFunction) {
    this.correlationService.setCorrelationId(req, res);
    next();
  }
}

// Usage in logger
class Logger {
  constructor(private readonly correlationService: CorrelationService) {}

  log(level: string, message: string, req?: Request, meta?: any): void {
    const correlationId = req
      ? this.correlationService.getCorrelationId(req)
      : undefined;

    console.log(JSON.stringify({
      timestamp: new Date().toISOString(),
      level,
      message,
      correlationId,
      ...meta
    }));
  }
}

Logging Patterns

1. Structured Logging

// logger.service.ts
interface LoggerConfig {
  level: string;
  format: 'json' | 'text';
  destination: 'console' | 'file' | 'stream';
  filePath?: string;
}

class LoggerService {
  private readonly logger: winston.Logger;

  constructor(private readonly config: LoggerConfig) {
    this.logger = this.createLogger();
  }

  private createLogger(): winston.Logger {
    const format = this.config.format === 'json'
      ? winston.format.json()
      : winston.format.combine(
          winston.format.timestamp(),
          winston.format.colorize(),
          winston.format.printf(
            ({ timestamp, level, message, ...meta }) =>
              `${timestamp} [${level}]: ${message} ${
                Object.keys(meta).length ? JSON.stringify(meta) : ''
              }`
          )
        );

    const transports = this.getTransports();

    return winston.createLogger({
      level: this.config.level,
      format,
      transports,
      defaultMeta: {
        service: 'api-service',
        environment: process.env.NODE_ENV
      }
    });
  }

  private getTransports(): winston.transport[] {
    const transports: winston.transport[] = [];

    if (this.config.destination === 'console') {
      transports.push(new winston.transports.Console());
    }

    if (this.config.destination === 'file' && this.config.filePath) {
      transports.push(
        new winston.transports.File({
          filename: this.config.filePath,
          maxsize: 5242880, // 5MB
          maxFiles: 5
        })
      );
    }

    return transports;
  }

  log(level: string, message: string, meta?: any): void {
    this.logger.log(level, message, meta);
  }

  error(message: string, error?: Error, meta?: any): void {
    this.logger.error(message, {
      error: error ? {
        message: error.message,
        stack: error.stack,
        name: error.name
      } : undefined,
      ...meta
    });
  }

  warn(message: string, meta?: any): void {
    this.logger.warn(message, meta);
  }

  info(message: string, meta?: any): void {
    this.logger.info(message, meta);
  }

  debug(message: string, meta?: any): void {
    this.logger.debug(message, meta);
  }
}

// Usage in service
@Injectable()
class UserService {
  constructor(
    private readonly logger: LoggerService,
    private readonly userRepository: UserRepository
  ) {}

  async createUser(userData: CreateUserDto): Promise<User> {
    this.logger.info('Creating new user', { email: userData.email });

    try {
      const user = await this.userRepository.create(userData);
      this.logger.info('User created successfully', {
        userId: user.id,
        email: user.email
      });
      return user;
    } catch (error) {
      this.logger.error('Failed to create user', error, {
        email: userData.email
      });
      throw error;
    }
  }
}

2. Log Aggregation

// log-aggregator.service.ts
interface LogAggregatorConfig {
  elasticsearch: {
    node: string;
    auth: {
      username: string;
      password: string;
    };
  };
  index: string;
  batchSize: number;
  flushInterval: number;
}

class LogAggregatorService {
  private readonly client: Client;
  private readonly buffer: any[];
  private readonly flushInterval: NodeJS.Timeout;

  constructor(private readonly config: LogAggregatorConfig) {
    this.client = new Client({
      node: this.config.elasticsearch.node,
      auth: this.config.elasticsearch.auth
    });

    this.buffer = [];
    this.flushInterval = setInterval(
      () => this.flush(),
      this.config.flushInterval
    );
  }

  async addLog(log: any): Promise<void> {
    this.buffer.push({
      ...log,
      timestamp: new Date().toISOString()
    });

    if (this.buffer.length >= this.config.batchSize) {
      await this.flush();
    }
  }

  private async flush(): Promise<void> {
    if (this.buffer.length === 0) {
      return;
    }

    const body = this.buffer.flatMap(log => [
      { index: { _index: this.config.index } },
      log
    ]);

    try {
      await this.client.bulk({ body });
      this.buffer.length = 0;
    } catch (error) {
      console.error('Failed to flush logs:', error);
    }
  }

  async search(query: any): Promise<any> {
    return this.client.search({
      index: this.config.index,
      body: query
    });
  }

  async close(): Promise<void> {
    clearInterval(this.flushInterval);
    await this.flush();
    await this.client.close();
  }
}

// Usage in logger
class AggregatedLogger {
  constructor(
    private readonly logger: LoggerService,
    private readonly aggregator: LogAggregatorService
  ) {}

  async log(level: string, message: string, meta?: any): Promise<void> {
    this.logger.log(level, message, meta);
    await this.aggregator.addLog({
      level,
      message,
      ...meta
    });
  }
}

Alerting Patterns

1. Alert Rules

// alert.service.ts
interface AlertConfig {
  rules: AlertRule[];
  notificationChannels: NotificationChannel[];
}

interface AlertRule {
  name: string;
  condition: (metrics: any) => boolean;
  severity: 'critical' | 'warning' | 'info';
  cooldown: number;
}

class AlertService {
  private readonly activeAlerts: Map<string, Alert>;
  private readonly metricsService: MetricsService;

  constructor(
    private readonly config: AlertConfig,
    metricsService: MetricsService
  ) {
    this.metricsService = metricsService;
    this.activeAlerts = new Map();
  }

  async checkAlerts(): Promise<void> {
    const metrics = await this.metricsService.getMetrics();

    for (const rule of this.config.rules) {
      const isTriggered = rule.condition(metrics);
      const alert = this.activeAlerts.get(rule.name);

      if (isTriggered && !alert) {
        await this.triggerAlert(rule);
      } else if (!isTriggered && alert) {
        await this.resolveAlert(rule);
      }
    }
  }

  private async triggerAlert(rule: AlertRule): Promise<void> {
    const alert: Alert = {
      name: rule.name,
      severity: rule.severity,
      triggeredAt: new Date(),
      resolvedAt: null
    };

    this.activeAlerts.set(rule.name, alert);

    await this.notifyChannels({
      type: 'alert',
      alert,
      message: `Alert triggered: ${rule.name}`
    });
  }

  private async resolveAlert(rule: AlertRule): Promise<void> {
    const alert = this.activeAlerts.get(rule.name);
    if (!alert) return;

    alert.resolvedAt = new Date();
    this.activeAlerts.delete(rule.name);

    await this.notifyChannels({
      type: 'resolution',
      alert,
      message: `Alert resolved: ${rule.name}`
    });
  }

  private async notifyChannels(notification: Notification): Promise<void> {
    await Promise.all(
      this.config.notificationChannels.map(channel =>
        channel.send(notification)
      )
    );
  }
}

// Usage
const alertService = new AlertService(
  {
    rules: [
      {
        name: 'high_error_rate',
        condition: (metrics) =>
          metrics.error_rate > 0.05,
        severity: 'critical',
        cooldown: 300000 // 5 minutes
      },
      {
        name: 'high_latency',
        condition: (metrics) =>
          metrics.p95_latency > 1000,
        severity: 'warning',
        cooldown: 600000 // 10 minutes
      }
    ],
    notificationChannels: [
      new EmailChannel({
        recipients: ['team@example.com']
      }),
      new SlackChannel({
        webhook: process.env.SLACK_WEBHOOK
      })
    ]
  },
  metricsService
);

// Start alert checking
setInterval(() => alertService.checkAlerts(), 60000);

2. Notification Channels

// notification.service.ts
interface NotificationConfig {
  channels: NotificationChannel[];
  templates: NotificationTemplate[];
}

class NotificationService {
  constructor(private readonly config: NotificationConfig) {}

  async sendNotification(
    type: string,
    data: any,
    channels: string[] = []
  ): Promise<void> {
    const template = this.config.templates.find(t => t.type === type);
    if (!template) {
      throw new Error(`Template not found for type: ${type}`);
    }

    const message = this.renderTemplate(template, data);
    const targetChannels = channels.length > 0
      ? this.config.channels.filter(c => channels.includes(c.type))
      : this.config.channels;

    await Promise.all(
      targetChannels.map(channel => channel.send(message))
    );
  }

  private renderTemplate(
    template: NotificationTemplate,
    data: any
  ): string {
    return template.content.replace(
      /\{\{(\w+)\}\}/g,
      (_, key) => data[key] || ''
    );
  }
}

// Channel implementations
class EmailChannel implements NotificationChannel {
  constructor(private readonly config: EmailConfig) {}

  async send(message: NotificationMessage): Promise<void> {
    await this.emailClient.send({
      to: this.config.recipients,
      subject: message.subject,
      text: message.text,
      html: message.html
    });
  }
}

class SlackChannel implements NotificationChannel {
  constructor(private readonly config: SlackConfig) {}

  async send(message: NotificationMessage): Promise<void> {
    await this.slackClient.chat.postMessage({
      channel: this.config.channel,
      text: message.text,
      blocks: message.blocks
    });
  }
}

class PagerDutyChannel implements NotificationChannel {
  constructor(private readonly config: PagerDutyConfig) {}

  async send(message: NotificationMessage): Promise<void> {
    await this.pagerDutyClient.incidents.create({
      incident: {
        type: 'incident',
        title: message.subject,
        urgency: message.severity === 'critical' ? 'high' : 'low',
        body: {
          type: 'incident_body',
          details: message.text
        },
        service: {
          id: this.config.serviceId,
          type: 'service_reference'
        }
      }
    });
  }
}

// Usage
const notificationService = new NotificationService({
  channels: [
    new EmailChannel({
      recipients: ['team@example.com']
    }),
    new SlackChannel({
      webhook: process.env.SLACK_WEBHOOK
    }),
    new PagerDutyChannel({
      serviceId: process.env.PAGERDUTY_SERVICE_ID
    })
  ],
  templates: [
    {
      type: 'alert',
      content: `
        Alert: {{name}}
        Severity: {{severity}}
        Message: {{message}}
        Time: {{time}}
      `
    },
    {
      type: 'resolution',
      content: `
        Alert Resolved: {{name}}
        Severity: {{severity}}
        Duration: {{duration}}
        Time: {{time}}
      `
    }
  ]
});

// Send notification
await notificationService.sendNotification(
  'alert',
  {
    name: 'high_error_rate',
    severity: 'critical',
    message: 'Error rate exceeded threshold',
    time: new Date().toISOString()
  },
  ['email', 'slack']
);

Best Practices

1. Metrics Collection

• Use appropriate metric types
• Define clear metric names
• Add relevant labels
• Regular metric review

2. Distributed Tracing

• Use consistent trace IDs
• Add meaningful spans
• Include relevant attributes
• Monitor trace quality

3. Logging

• Use structured logging
• Include context
• Set appropriate levels
• Regular log review

4. Alerting

• Define clear thresholds
• Use appropriate severity
• Implement cooldowns
• Regular alert review

Conclusion

Advanced monitoring patterns enable building observable, maintainable applications. By understanding and applying these patterns, developers can create robust monitoring solutions that provide insights into application behavior.

Resources

• Monitoring Best Practices
• Metrics Guide
• Tracing Guide
• Logging Guide