Scraping Historical X (Twitter) Posts via Firefox Console

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
 
package main

import (
	"encoding/json"
	"fmt"
	"os"
	"regexp"
	"strings"
)

// RawItem represents the structure of the input JSON
type RawItem struct {
	Date     string `json:"date"`
	TweetURL string `json:"tweet_url"`
	Text     string `json:"text"`
	Links    string `json:"links"`
}

// CleanItem represents the desired cleaned output schema
type CleanItem struct {
	Date        string `json:"date"`
	VideoLink   string `json:"video_link"`
	Description string `json:"description"`
}

func main() {
	// 1. Read raw JSON file
	fileData, err := os.ReadFile("raw_tweets.json")
	if err != nil {
		fmt.Printf("Error reading input file: %v\n", err)
		return
	}

	var rawItems []RawItem
	if err := json.Unmarshal(fileData, &rawItems); err != nil {
		fmt.Printf("Error unmarshaling JSON: %v\n", err)
		return
	}

	// 2. Compile Regex to catch YouTube links (including those with spaces)
	ytRegex := regexp.MustCompile(`(?i)https?://\s*(?:www\.)?(?:youtube\.com/[^\s]+|youtu\.be/[^\s]+)`)
	spaceRegex := regexp.MustCompile(`\s+`)

	var cleanedData []CleanItem

	// 3. Process items
	for _, item := range rawItems {
		match := ytRegex.FindString(item.Text)
		if match != "" {
			// Remove spaces inside the matched URL
			videoLink := strings.ReplaceAll(match, " ", "")

			// Clean description by removing the URL and collapsing extra whitespace
			description := ytRegex.ReplaceAllString(item.Text, "")
			description = strings.TrimSpace(spaceRegex.ReplaceAllString(description, " "))

			cleanedData = append(cleanedData, CleanItem{
				Date:        item.Date,
				VideoLink:   videoLink,
				Description: description,
			})
		}
	}

	// 4. Export cleaned JSON
	outputJSON, err := json.MarshalIndent(cleanedData, "", "  ")
	if err != nil {
		fmt.Printf("Error marshaling output JSON: %v\n", err)
		return
	}

	if err := os.WriteFile("cleaned_videos.json", outputJSON, 0644); err != nil {
		fmt.Printf("Error writing file: %v\n", err)
		return
	}

	fmt.Printf("Success! Extracted %d valid YouTube videos.\n", len(cleanedData))
}
 

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
 
/*
eda_analysis.go
Perform dataset exploration and word count analysis on parsed video metadata.
*/

package main

import (
	"encoding/json"
	"fmt"
	"os"
	"regexp"
	"sort"
	"strings"
	"time"
)

type VideoRecord struct {
	Date        string `json:"date"`
	VideoLink   string `json:"video_link"`
	Description string `json:"description"`
}

type WordCount struct {
	Word  string
	Count int
}

func main() {
	// 1. Read Cleaned JSON
	fileData, err := os.ReadFile("cleaned_videos.json")
	if err != nil {
		fmt.Printf("Error reading file: %v\n", err)
		return
	}

	var records []VideoRecord
	if err := json.Unmarshal(fileData, &records); err != nil {
		fmt.Printf("Error unmarshaling JSON: %v\n", err)
		return
	}

	if len(records) == 0 {
		fmt.Println("No records to process.")
		return
	}

	// 2. Variables for Stats
	var totalWords int
	minWords, maxWords := 1<<31-1, 0
	var minTime, maxTime time.Time
	wordMap := make(map[string]int)
	wordRegex := regexp.MustCompile(`\b\w+\b`)
	stopWords := map[string]bool{"the": true, "a": true, "to": true, "and": true, "of": true, "in": true, "is": true, "on": true, "for": true, "this": true}

	// 3. Iterative Analysis
	for i, r := range records {
		t, _ := time.Parse(time.RFC3339, r.Date)
		if i == 0 || t.Before(minTime) {
			minTime = t
		}
		if i == 0 || t.After(maxTime) {
			maxTime = t
		}

		// Text Stats
		words := wordRegex.FindAllString(strings.ToLower(r.Description), -1)
		wc := len(words)
		totalWords += wc

		if wc < minWords {
			minWords = wc
		}
		if wc > maxWords {
			maxWords = wc
		}

		// Track keywords
		for _, w := range words {
			if !stopWords[w] && len(w) > 2 {
				wordMap[w]++
			}
		}
	}

	// 4. Sort Top Keywords
	var wordCounts []WordCount
	for k, v := range wordMap {
		wordCounts = append(wordCounts, WordCount{Word: k, Count: v})
	}
	sort.Slice(wordCounts, func(i, j int) bool {
		return wordCounts[i].Count > wordCounts[j].Count
	})

	// 5. Print Results
	fmt.Println("================ EDA SUMMARY REPORT ================")
	fmt.Printf("Total Videos Analyzed : %d\n", len(records))
	fmt.Printf("Date Range           : %s to %s\n", minTime.Format("2006-01-02"), maxTime.Format("2006-01-02"))
	fmt.Println("\n--- Description Text Metrics ---")
	fmt.Printf("Avg Word Count       : %.2f words\n", float64(totalWords)/float64(len(records)))
	fmt.Printf("Max Word Count       : %d words\n", maxWords)
	fmt.Printf("Min Word Count       : %d words\n", minWords)
	fmt.Println("\n--- Top Keywords / Themes ---")
	for i := 0; i < 5 && i < len(wordCounts); i++ {
		fmt.Printf(" - %s: %d occurrences\n", wordCounts[i].Word, wordCounts[i].Count)
	}
	fmt.Println("====================================================")
}