Youjourney

Youjourney

I’m Codin’ It

셀레니움 크롤러와 코틀린으로 만든 백준 문제풀이 마크다운 자동생성기

개요

깃허브 블로그를 하면서 마크다운을 일일이 작성하기가 번거롭고 귀찮아서 만들었습니다.

1 문제를 확인한다.
2 문제를 푼다.
3 채점을 한다. (성공하지 못했을 경우 2번으로…….)
4 마크다운을 작성한다.
4 새로 만든 크롤러에 크롤링부터 마크다운 작성까지 맡긴다.
5 jykell로 마크다운을 제네레이팅한다.
6 로컬에서 의도한대로 웹페이지가 생성되었는지 확인한다.
7 깃허브에 커밋 및 푸시한다.

문제의 소지가 있다면 좌측의 이메일로 연락주시면 감사하겠습니다.

개발환경

macOS 11.1 (Intel Mac) 기준

인텔리제이(IntelliJ 2020.3)
OpenJDK-15.0.1
Microsoft Edge Browser(87.0.664.66)
Microsoft Edge WebDriver(87.0.664.66)
selenium-java (4.0.0-alpha-7)

기타 셀레니움 사용관련 정보

셀레니움 크롤러 설치 및 세팅방법은 여기

소스파일

Crawler.kt(ver.20210112-1)

주의
예측 가능한 오류가 있음. (마크다운 바디에 “-“가 있을 경우 지킬에서 문서가 제대로 생성되지 않을 수 있습니다.)
문제를 풀면서 마크다운이 제대로 생성되지 않는 오류가 발생할 때 수정할 예정

import org.openqa.selenium.By
import org.openqa.selenium.WebDriverException
import org.openqa.selenium.edge.EdgeDriver
import org.openqa.selenium.edge.EdgeOptions
import java.io.File
import java.io.FileReader
import java.io.FileWriter
import java.io.BufferedReader
import java.io.BufferedWriter
import java.lang.Exception
import java.text.SimpleDateFormat
import java.util.Date
import java.util.Calendar


class BOJCrawler(private val problem: String, private val doubleCheck: Boolean) {
    private val webDriverID = "webdriver.edge.driver"
    private val webDriverPath = "웹드라이버의 경로"
    //private val webDriverPath = "/Users/사용자폴더/Downloads/edgedriver_mac64/msedgedriver"
    private val options: EdgeOptions = EdgeOptions()
    private val driver: EdgeDriver by lazy { EdgeDriver(options) }

    init {
        println("init. Crawler initializing...")
        System.setProperty(webDriverID, webDriverPath)
        options.addArguments("headless")
        options.addArguments("window-size=1920x1080")
        options.addArguments("disable-gpu")
        options.addArguments("disable-infobars")
        options.addArguments("--disable-extensions")
        options.addArguments("--disable-default-apps")

        getProblem()
    }

    private fun getProblem() {
        val url = "https://www.acmicpc.net/problem/"
        println("Crawling from ${url}${problem}")

        try {
            println("Loading Edge driver...")
            driver.get("${url}${problem}")
        } catch (e: WebDriverException) {
            println(e)
        }
    }

    private fun makeMDFrontMatters(): String {
        println("Building markdown file front matters...")
        val strArr = mutableListOf<String>()

        //begin frontmatters
        strArr.add("---")

        //title
        strArr.add(
            let {
                val title = driver
                    .title
                    .split("번:")
                    .map { it.trim() }

                "title: \"[백준] ${title[1]} (${title[0]})(kotlin)\""
            }
        )

        //excerpt
        strArr.add(
            let {
                val description = driver
                    .findElement(By.xpath("/html/body/div[2]/div[2]/div[3]/div[5]/div[1]/section/div[2]"))
                    .text
                    .split(".")[0]
                    .replace("\n", " ") + "."

                "excerpt: \"$description\""
            }
        )

        //categories
        strArr.add("categories:\n- boj")

        //tags
        strArr.add("tags:\n- algorithm\n- kotlin")

        //last_modified_at
        strArr.add(
            let {
                val currentDateTime: Date = Calendar.getInstance().time
                val dateFormat: SimpleDateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss+09:00")

                "last_modified_at: ${dateFormat.format(currentDateTime)}"
            }
        )

        //end of frontmatters
        strArr.add("---")
        return strArr.joinToString("\n", "", "\n")
    }

    private fun makeMDBody(): String {
        println("Building markdown body...")
        val strArr = mutableListOf<String>()

        fun br3() = strArr.add("\n\n\n")
        fun br2() = strArr.add("\n\n")
        fun br1() = strArr.add("\n")

        //문제 설명
        br1()
        strArr.add("### 문제 설명")
        strArr.add("[백준 ${problem}번 문제 링크](https://www.acmicpc.net/problem/${problem}#description)")
        br3()

        //문제 입출력 설명
        val problem_input = driver.findElement(By.id("problem_input")).text
        val problem_output = driver.findElement(By.id("problem_output")).text
        strArr.add("### 입력 및 출력")
        if (problem_input != "") {
            strArr.add("#### >> 입력")
            strArr.add(driver.findElement(By.id("problem_input")).text)
            br2()
        }
        if (problem_output != "") {
            strArr.add("#### >> 출력")
            strArr.add(driver.findElement(By.id("problem_output")).text)
            br2()
        }
        br1()

        //문제 제한 설명
        val problem_limit = driver.findElement(By.id("problem_limit")).text
        if (problem_limit != "") {
            strArr.add("### 제한 사항")
            br1()
            strArr.add(problem_limit)
            br3()
        }

        //문제 예제 입출력
        var sample_input_count = 1
        while (true) {
            try {
                driver.findElement(By.id("sample-input-${sample_input_count}"))
                sample_input_count++
            } catch (e: WebDriverException) {
                 sample_input_count--
                break
            }
        }

        if (sample_input_count != 0) {
            strArr.add("### 예제 입출력")
            br1()
            strArr.add("|입력|출력|")
            strArr.add("|-----|------|")
            for (count in 1..sample_input_count) {
                val input = driver.findElement(By.id("sample-input-$count"))
                    .text
                    .let {
                        it.replace("\n", "<br>")
                    }
                val output = driver.findElement(By.id("sample-output-$count"))
                    .text
                    .let {
                        it.replace("\n", "<br>")
                    }
                strArr.add("|$input|$output|")
            }
            br3()
        }

        //문제 힌트
        val problem_hint = driver.findElement(By.id("problem_hint")).text
        if (problem_hint != "") {
            strArr.add("### 문제 힌트")
            br1()
            strArr.add(problem_hint)
            br3()
        }

        driver.close()
        return strArr.joinToString("\n", "", "\n")
    }

    private fun loadKtSource() : String {
        println("Loading Kotlin source file...")
        val str = mutableListOf<String>()
        val path = "문제를 푼 코틀린 소스파일의 경로"
        //val path = "/Users/사용자폴더/Documents/GIT_ProblemSolutions/ProblemSolutions/BOJ${problem}/src/main/kotlin/main.kt"

        try {
            val file = File(path)
            val fr = FileReader(file)
            val br = BufferedReader(fr)

            str.add("### 문제 풀이1")
            str.add("```kotlin")
            str.addAll(br.readLines())
            str.add("```")

            br.close()
            fr.close()
        } catch (e: Exception) {
            e.printStackTrace()
        }

        return str.joinToString("\n")
    }

    fun makeMDFile() {
        val currentDateTime: Date = Calendar.getInstance().time
        val dateFormat: SimpleDateFormat = SimpleDateFormat("yyyy-MM-dd")
        val path = "마크다운 파일이 저장될 폴더 위치"
        //val path = "/Users/사용자폴더/Documents/GIT_youjourney.github.io/_posts/"
        val fileName = "${dateFormat.format(currentDateTime)}-BOJ${problem}.md"
        println("Making ${fileName}...")

        try {
            val fw = FileWriter(path + fileName)
            val bw = BufferedWriter(fw)

            bw.write(makeMDFrontMatters() + makeMDBody() + loadKtSource())
            bw.close()
            fw.close()
        } catch (e: Exception) {
            println("Couldn't make the ${dateFormat.format(currentDateTime)}-${problem}.md successfully.")
            e.printStackTrace()
        } finally {
            println(fileName + "is made successfully!")
            //println("File is written at : /Users/사용자명/Documents/GIT_youjourney.github.io/_posts/${dateFormat.format(currentDateTime)}-${problem}.md")
        }

        if (doubleCheck) {
            println("Double Check is enabled. Checking now.")
            println("======================================")
            try {
                val file = File(path + fileName)
                val fr = FileReader(file)
                val br = BufferedReader(fr)

                br.forEachLine { println(it) }
                br.close()
                fr.close()
            } catch (e: Exception) {
                e.printStackTrace()
            }
        }
    }
}

fun main(args: Array<String>) {
    val doubleCheck = true
    val crawler: BOJCrawler = BOJCrawler("17362", doubleCheck)
    crawler.makeMDFile()
}

Crawler.kt(ver.20210112-2)

알림
마크다운 문서 내부에 -표시에 대한 오류 수정

package com

import org.openqa.selenium.By
import org.openqa.selenium.WebDriverException
import org.openqa.selenium.edge.EdgeDriver
import org.openqa.selenium.edge.EdgeOptions
import java.io.File
import java.io.FileReader
import java.io.FileWriter
import java.io.BufferedReader
import java.io.BufferedWriter
import java.text.SimpleDateFormat
import java.util.Date
import java.util.Calendar
import kotlin.Exception


class BOJCrawler(private val problem: String, private val doubleCheck: Boolean) {
    private val webDriverID = "webdriver.edge.driver"
    private val webDriverPath = "웹드라이버의 경로"
    //private val webDriverPath = "/Users/사용자폴더/Downloads/edgedriver_mac64/msedgedriver"
    private val options: EdgeOptions = EdgeOptions()
    private val driver: EdgeDriver by lazy { EdgeDriver(options) }

    init {
        println("init. Crawler initializing...")
        System.setProperty(webDriverID, webDriverPath)
        options.addArguments("headless")
        options.addArguments("window-size=1920x1080")
        options.addArguments("disable-gpu")
        options.addArguments("disable-infobars")
        options.addArguments("--disable-extensions")
        options.addArguments("--disable-default-apps")

        getProblem()
    }

    private fun getProblem() {
        val url = "https://www.acmicpc.net/problem/"
        println("Crawling from ${url}${problem}")


        try {
            println("Loading Edge driver...")
            driver.get("${url}${problem}")
        } catch (e: WebDriverException) {
            println(e)
        }
    }

    private fun makeMDFrontMatters(): String {
        println("Building markdown file front matters...")
        val strArr = mutableListOf<String>()

        //begin frontmatters
        strArr.add("---")

        //title
        strArr.add(
            let {
                val title = driver
                    .title
                    .split("번:")
                    .map { it.trim() }

                "title: \"[백준] ${title[1]} (${title[0]})(kotlin)\""
            }
        )

        //excerpt
        strArr.add(
            let {
                val description = driver
                    .findElement(By.xpath("/html/body/div[2]/div[2]/div[3]/div[5]/div[1]/section/div[2]"))
                    .text
                    .split(".")[0]
                    .replace("\n", " ") + "."

                "excerpt: \"$description\""
            }
        )

        //categories
        strArr.add("categories:\n- boj")

        //tags
        strArr.add("tags:\n- algorithm\n- kotlin")

        //last_modified_at
        strArr.add(
            let {
                val currentDateTime: Date = Calendar.getInstance().time
                val dateFormat: SimpleDateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss+09:00")

                "last_modified_at: ${dateFormat.format(currentDateTime)}"
            }
        )

        //end of frontmatters
        strArr.add("---")
        return strArr.joinToString("\n", "", "\n")
    }

    private fun makeMDBody(): String {
        println("Building markdown body...")
        val strArr = mutableListOf<String>()

        fun br3() = strArr.add("\n\n\n")
        fun br2() = strArr.add("\n\n")
        fun br1() = strArr.add("\n")

        //문제 설명
        br1()
        strArr.add("### 문제 설명")
        strArr.add("[백준 ${problem}번 문제 링크](https://www.acmicpc.net/problem/${problem}#description)")
        br3()

        //문제 입출력 설명
        val problem_input = driver.findElement(By.id("problem_input")).text
        val problem_output = driver.findElement(By.id("problem_output")).text
        strArr.add("### 입력 및 출력")
        if (problem_input != "") {
            strArr.add("#### >> 입력")
            strArr.add(driver.findElement(By.id("problem_input")).text)
            br2()
        }
        if (problem_output != "") {
            strArr.add("#### >> 출력")
            strArr.add(driver.findElement(By.id("problem_output")).text)
            br2()
        }
        br1()

        //문제 제한 설명
        val problem_limit = driver.findElement(By.id("problem_limit")).text
        if (problem_limit != "") {
            strArr.add("### 제한 사항")
            br1()
            strArr.add(problem_limit)
            br3()
        }

        //문제 예제 입출력
        var sample_input_count = 1
        while (true) {
            try {
                driver.findElement(By.id("sample-input-${sample_input_count}"))
                sample_input_count++
            } catch (e: WebDriverException) {
                sample_input_count--
                break
            }
        }

        if (sample_input_count != 0) {
            strArr.add("### 예제 입출력")
            br1()
            strArr.add("|입력|출력|")
            strArr.add("|-----|------|")
            for (count in 1..sample_input_count) {
                val input = driver.findElement(By.id("sample-input-$count"))
                    .text
                    .let {
                        it.replace("\n", "<br>")
                    }
                val output = driver.findElement(By.id("sample-output-$count"))
                    .text
                    .let {
                        it.replace("\n", "<br>")
                    }
                strArr.add("|$input|$output|")
            }
            br3()
        }

        //문제 힌트
        val problem_hint = driver.findElement(By.id("problem_hint")).text
        if (problem_hint != "") {
            strArr.add("### 문제 힌트")
            br1()
            strArr.add(problem_hint)
            br3()
        }

        driver.close()
        return strArr.joinToString("\n", "", "\n")
    }

    private fun loadKtSource() : String {
        println("Loading Kotlin source file...")
        val str = mutableListOf<String>()
        val path = "문제를 푼 코틀린 소스파일의 경로"
        //val path = "/Users/사용자폴더/Documents/GIT_ProblemSolutions/ProblemSolutions/BOJ${problem}/src/main/kotlin/main.kt"

        try {
            val file = File(path)
            val fr = FileReader(file)
            val br = BufferedReader(fr)

            str.add("### 문제 풀이1")
            str.add("```kotlin")
            str.addAll(br.readLines())
            str.add("```")

            br.close()
            fr.close()
        } catch (e: Exception) {
            e.printStackTrace()
        }

        return str.joinToString("\n")
    }

    fun makeMDFile() {
        val currentDateTime: Date = Calendar.getInstance().time
        val dateFormat: SimpleDateFormat = SimpleDateFormat("yyyy-MM-dd")
        val path = "마크다운 파일이 저장될 폴더 위치"
        //val path = "/Users/사용자폴더/Documents/GIT_youjourney.github.io/_posts/"
        val fileName = "${dateFormat.format(currentDateTime)}-BOJ${problem}.md"
        println("Making ${fileName}...")

        try {
            val fw = FileWriter(path + fileName)
            val bw = BufferedWriter(fw)

            bw.write(makeMDFrontMatters() + makeMDBody() + loadKtSource())
            bw.close()
            fw.close()
        } catch (e: Exception) {
            println("Can't write the ${dateFormat.format(currentDateTime)}-${problem}.md")
            e.printStackTrace()
        } finally {
            println("$fileName is made successfully!")
            //println("File is written at : /Users/사용자명/Documents/GIT_youjourney.github.io/_posts/${dateFormat.format(currentDateTime)}-${problem}.md")
        }

        if (doubleCheck) {
            println("Double Check is enabled. Checking now.")
            println("======================================")
            try {
                val file = File(path + fileName)
                val fr = FileReader(file)
                val br = BufferedReader(fr)

                br.forEachLine { println(it) }
                br.close()
                fr.close()
            } catch (e: Exception) {
                e.printStackTrace()
            }
        }
    }
}

fun main(args: Array<String>) {
    val doubleCheck: Boolean = args[1]
        .let {
            when (it.toLowerCase()) {
                "true" -> true
                else -> false
            }
        }
    val crawler: BOJCrawler = BOJCrawler(args[0], doubleCheck)

    crawler.makeMDFile()
}

Crawler.kt(ver.20210113-1)

package com

import org.openqa.selenium.By
import org.openqa.selenium.WebDriverException
import org.openqa.selenium.edge.EdgeDriver
import org.openqa.selenium.edge.EdgeOptions
import java.io.File
import java.io.FileReader
import java.io.FileWriter
import java.io.BufferedReader
import java.io.BufferedWriter
import java.text.SimpleDateFormat
import java.util.Date
import java.util.Calendar
import kotlin.Exception


class BOJCrawler(private val problem: String, private val doubleCheck: Boolean) {
    private val webDriverID = "webdriver.edge.driver"
    private val webDriverPath = "웹드라이버의 경로"
    //private val webDriverPath = "/Users/사용자폴더/Downloads/edgedriver_mac64/msedgedriver"
    private val options: EdgeOptions = EdgeOptions()
    private val driver: EdgeDriver by lazy { EdgeDriver(options) }

    init {
        println("init. Crawler initializing...")
        System.setProperty(webDriverID, webDriverPath)
        options.addArguments("headless")
        options.addArguments("window-size=1920x1080")
        options.addArguments("disable-gpu")
        options.addArguments("disable-infobars")
        options.addArguments("--disable-extensions")
        options.addArguments("--disable-default-apps")

        getProblem()
    }

    private fun getProblem() {
        val url = "https://www.acmicpc.net/problem/"
        println("Crawling from ${url}${problem}")


        try {
            println("Loading Edge driver...")
            driver.get("${url}${problem}")
        } catch (e: WebDriverException) {
            println(e)
        }
    }

    private fun makeMDFrontMatters(): String {
        println("Building markdown file front matters...")
        val strArr = mutableListOf<String>()

        //begin frontmatters
        strArr.add("---")

        //title
        strArr.add(
            let {
                val title = driver
                    .title
                    .split("번:")
                    .map { it.trim() }

                "title: \"[백준] ${title[1]} (${title[0]})(kotlin)\""
            }
        )

        //excerpt
        strArr.add(
            let {
                val description = driver
                    .findElement(By.xpath("/html/body/div[2]/div[2]/div[3]/div[5]/div[1]/section/div[2]"))
                    .text
                    .split(".")[0]
                    .replace("\n", " ") + "."

                "excerpt: \"$description\""
            }
        )

        //categories
        strArr.add("categories:\n- boj")

        //tags
        strArr.add("tags:\n- algorithm\n- kotlin")

        //last_modified_at
        strArr.add(
            let {
                val currentDateTime: Date = Calendar.getInstance().time
                val dateFormat: SimpleDateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss+09:00")

                "last_modified_at: ${dateFormat.format(currentDateTime)}"
            }
        )

        //end of frontmatters
        strArr.add("---")
        return strArr.joinToString("\n", "", "\n")
    }

    private fun makeMDBody(): String {
        println("Building markdown body...")
        val strArr = mutableListOf<String>()

        fun br3() = strArr.add("\n\n\n")
        fun br2() = strArr.add("\n\n")
        fun br1() = strArr.add("\n")

        //문제 설명
        br1()
        strArr.add("### 문제 설명")
        strArr.add("[백준 ${problem}번 문제 링크](https://www.acmicpc.net/problem/${problem}#description)")
        br3()

        //문제 입출력 설명
        val problem_input = driver.findElement(By.id("problem_input")).text
        val problem_output = driver.findElement(By.id("problem_output")).text
        strArr.add("### 입력 및 출력")
        if (problem_input != "") {
            strArr.add("#### >> 입력")
            strArr.add(driver.findElement(By.id("problem_input")).text.replace("-", "\\-"))
            br2()
        }
        if (problem_output != "") {
            strArr.add("#### >> 출력")
            strArr.add(driver.findElement(By.id("problem_output")).text.replace("-", "\\-"))
            br2()
        }
        br1()

        //문제 제한 설명
        val problem_limit = driver.findElement(By.id("problem_limit")).text
        if (problem_limit != "") {
            strArr.add("### 제한 사항")
            br1()
            strArr.add(problem_limit.replace("-", "\\-"))
            br3()
        }

        //문제 예제 입출력
        var sample_input_count = 1
        while (true) {
            try {
                driver.findElement(By.id("sample-input-${sample_input_count}"))
                sample_input_count++
            } catch (e: WebDriverException) {
                sample_input_count--
                break
            }
        }

        if (sample_input_count != 0) {
            strArr.add("### 예제 입출력")
            br1()
            strArr.add("|입력|출력|")
            strArr.add("|-----|------|")
            for (count in 1..sample_input_count) {
                val input = driver.findElement(By.id("sample-input-$count"))
                    .text
                    .let {
                        it.replace("\n", "<br>")
                    }
                    .replace("-", "\\-")
                val output = driver.findElement(By.id("sample-output-$count"))
                    .text
                    .let {
                        it.replace("\n", "<br>")
                    }
                    .replace("-", "\\-")
                strArr.add("|$input|$output|")
            }
            br3()
        }

        //문제 힌트
        val problem_hint = driver.findElement(By.id("problem_hint")).text
        if (problem_hint != "") {
            strArr.add("### 문제 힌트")
            br1()
            strArr.add(problem_hint.replace("-", "\\-"))
            br3()
        }

        driver.close()
        return strArr.joinToString("\n", "", "\n")
    }

    fun makeMDFile() {
        val currentDateTime: Date = Calendar.getInstance().time
        val dateFormat: SimpleDateFormat = SimpleDateFormat("yyyy-MM-dd")
        val path = "마크다운 파일이 저장될 폴더 위치"
        //val path = "/Users/사용자폴더/Documents/GIT_youjourney.github.io/_posts/"
        val fileName = "${dateFormat.format(currentDateTime)}-BOJ${problem}.md"
        println("Making ${fileName}...")

        try {
            val fw = FileWriter(path + fileName)
            val bw = BufferedWriter(fw)

            bw.write(makeMDFrontMatters() + makeMDBody() + loadKtSource())
            bw.close()
            fw.close()
        } catch (e: Exception) {
            println("Can't write the ${dateFormat.format(currentDateTime)}-${problem}.md")
            e.printStackTrace()
        } finally {
            println("$fileName is made successfully!")
            //println("File is written at : /Users/사용자명/Documents/GIT_youjourney.github.io/_posts/${dateFormat.format(currentDateTime)}-${problem}.md")
        }

        if (doubleCheck) {
            println("Double Check is enabled. Checking now.")
            println("======================================")
            try {
                val file = File(path + fileName)
                val fr = FileReader(file)
                val br = BufferedReader(fr)

                br.forEachLine { println(it) }
                br.close()
                fr.close()
            } catch (e: Exception) {
                e.printStackTrace()
            }
        }
    }
}

fun main(args: Array<String>) {
    val doubleCheck: Boolean = args[1]
        .let {
            when (it.toLowerCase()) {
                "true" -> true
                else -> false
            }
        }
    val crawler: BOJCrawler = BOJCrawler(args[0], doubleCheck)

    crawler.makeMDFile()
}

Crawler.kt(ver.20210113-2)

알림

  • 마크다운 frontmatter의 excerpt에 "가 포함될 경우 에러가 발생하던 문제 수정
  • 입력 및 출력 설명에서 한 문장에 여러줄로 나누어지는 문제 수정
package com

import org.openqa.selenium.By
import org.openqa.selenium.WebDriverException
import org.openqa.selenium.edge.EdgeDriver
import org.openqa.selenium.edge.EdgeOptions
import java.io.File
import java.io.FileReader
import java.io.FileWriter
import java.io.BufferedReader
import java.io.BufferedWriter
import java.text.SimpleDateFormat
import java.util.Date
import java.util.Calendar
import kotlin.Exception


class BOJCrawler(private val problem: String, private val doubleCheck: Boolean) {
    private val webDriverID = "webdriver.edge.driver"
    private val webDriverPath = "웹드라이버의 경로"
    //private val webDriverPath = "/Users/사용자폴더/Downloads/edgedriver_mac64/msedgedriver"
    private val options: EdgeOptions = EdgeOptions()
    private val driver: EdgeDriver by lazy { EdgeDriver(options) }

    init {
        println("init. Crawler initializing...")
        System.setProperty(webDriverID, webDriverPath)
        options.addArguments("headless")
        options.addArguments("window-size=1920x1080")
        options.addArguments("disable-gpu")
        options.addArguments("disable-infobars")
        options.addArguments("--disable-extensions")
        options.addArguments("--disable-default-apps")

        getProblem()
    }

    private fun getProblem() {
        val url = "https://www.acmicpc.net/problem/"
        println("Crawling from ${url}${problem}")


        try {
            println("Loading Edge driver...")
            driver.get("${url}${problem}")
        } catch (e: WebDriverException) {
            println(e)
        }
    }

    private fun makeMDFrontMatters(): String {
        println("Building markdown file front matters...")
        val strArr = mutableListOf<String>()

        //begin frontmatters
        strArr.add("---")

        //title
        strArr.add(
            let {
                val title = driver
                    .title
                    .split("번:")
                    .map { it.trim() }

                "title: \"[백준] ${title[1]} (${title[0]})(kotlin)\""
            }
        )

        //excerpt
        strArr.add(
            let {
                val description = driver
                    .findElement(By.xpath("/html/body/div[2]/div[2]/div[3]/div[5]/div[1]/section/div[2]"))
                    .text
                    .split(".")[0]
                    .replace("\"", "\\\"")
                    .replace("\n", " ") + "."

                "excerpt: \"$description\""
            }
        )

        //categories
        strArr.add("categories:\n- boj")

        //tags
        strArr.add("tags:\n- algorithm\n- kotlin")

        //last_modified_at
        strArr.add(
            let {
                val currentDateTime: Date = Calendar.getInstance().time
                val dateFormat: SimpleDateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss+09:00")

                "last_modified_at: ${dateFormat.format(currentDateTime)}"
            }
        )

        //end of frontmatters
        strArr.add("---")
        return strArr.joinToString("\n", "", "\n")
    }

    private fun makeMDBody(): String {
        println("Building markdown body...")
        val strArr = mutableListOf<String>()

        fun br3() = strArr.add("\n\n\n")
        fun br2() = strArr.add("\n\n")
        fun br1() = strArr.add("\n")

        //문제 설명
        br1()
        strArr.add("### 문제 설명")
        strArr.add("[백준 ${problem}번 문제 링크](https://www.acmicpc.net/problem/${problem}#description)")
        br3()

        //문제 입출력 설명
        val problem_input = driver.findElement(By.id("problem_input"))
            .findElements(By.tagName("p"))
            .map {
                it
                    .text
                    .replace("\n", " ")
                    .replace("-", "\\-")
            }
        val problem_output = driver.findElement(By.id("problem_output"))
            .findElements(By.tagName("p"))
            .map {
                it
                    .text
                    .replace("\n", " ")
                    .replace("-", "\\-")
            }
        strArr.add("### 입력 및 출력")
        if (problem_input[0] != "") {
            strArr.add("#### >> 입력")
            if (problem_input.size == 1) {
                strArr.addAll(problem_input)
            } else {
                strArr.add(problem_input.joinToString("\n", "* "))
            }
            br2()
        }
        if (problem_output[0] != "") {
            strArr.add("#### >> 출력")
            if (problem_output.size == 1) {
                strArr.addAll(problem_output)
            } else {
                strArr.add(problem_output.joinToString("\n", "* "))
            }
            br2()
        }
        br1()

        //문제 제한 설명
        val problem_limit = driver.findElement(By.id("problem_limit")).text
        if (problem_limit != "") {
            strArr.add("### 제한 사항")
            br1()
            strArr.add(problem_limit.replace("-", "\\-"))
            br3()
        }

        //문제 예제 입출력
        var sample_input_count = 1
        while (true) {
            try {
                driver.findElement(By.id("sample-input-${sample_input_count}"))
                sample_input_count++
            } catch (e: WebDriverException) {
                sample_input_count--
                break
            }
        }

        if (sample_input_count != 0) {
            strArr.add("### 예제 입출력")
            br1()
            strArr.add("|입력|출력|")
            strArr.add("|-----|------|")
            for (count in 1..sample_input_count) {
                val input = driver.findElement(By.id("sample-input-$count"))
                    .text
                    .let {
                        it.replace("\n", "<br>")
                    }
                    .replace("-", "\\-")
                val output = driver.findElement(By.id("sample-output-$count"))
                    .text
                    .let {
                        it.replace("\n", "<br>")
                    }
                    .replace("-", "\\-")
                strArr.add("|$input|$output|")
            }
            br3()
        }

        //문제 힌트
        val problem_hint = driver.findElement(By.id("problem_hint")).text
        if (problem_hint != "") {
            strArr.add("### 문제 힌트")
            br1()
            strArr.add(problem_hint.replace("-", "\\-"))
            br3()
        }

        driver.close()
        return strArr.joinToString("\n", "", "\n")
    }

    fun makeMDFile() {
        val currentDateTime: Date = Calendar.getInstance().time
        val dateFormat: SimpleDateFormat = SimpleDateFormat("yyyy-MM-dd")
        val path = "마크다운 파일이 저장될 폴더 위치"
        //val path = "/Users/사용자폴더/Documents/GIT_youjourney.github.io/_posts/"
        val fileName = "${dateFormat.format(currentDateTime)}-BOJ${problem}.md"
        println("Making ${fileName}...")

        try {
            val fw = FileWriter(path + fileName)
            val bw = BufferedWriter(fw)

            bw.write(makeMDFrontMatters() + makeMDBody() + loadKtSource())
            bw.close()
            fw.close()
        } catch (e: Exception) {
            println("Can't write the ${dateFormat.format(currentDateTime)}-${problem}.md")
            e.printStackTrace()
        } finally {
            println("$fileName is made successfully!")
            //println("File is written at : /Users/사용자명/Documents/GIT_youjourney.github.io/_posts/${dateFormat.format(currentDateTime)}-${problem}.md")
        }

        if (doubleCheck) {
            println("Double Check is enabled. Checking now.")
            println("======================================")
            try {
                val file = File(path + fileName)
                val fr = FileReader(file)
                val br = BufferedReader(fr)

                br.forEachLine { println(it) }
                br.close()
                fr.close()
            } catch (e: Exception) {
                e.printStackTrace()
            }
        }
    }
}

fun main(args: Array<String>) {
    val doubleCheck: Boolean = args[1]
        .let {
            when (it.toLowerCase()) {
                "true" -> true
                else -> false
            }
        }
    val crawler: BOJCrawler = BOJCrawler(args[0], doubleCheck)

    crawler.makeMDFile()
}

Crawler.kt(ver.20210113-3)

알림

  • 백준 문제설명에서 첫 줄이 글자가 아닌 경우 마크다운 frontmatter에 excerpt에 내용이 채워지지 않는 문제를 수정
package com

import org.openqa.selenium.By
import org.openqa.selenium.WebDriverException
import org.openqa.selenium.edge.EdgeDriver
import org.openqa.selenium.edge.EdgeOptions
import java.io.File
import java.io.FileReader
import java.io.FileWriter
import java.io.BufferedReader
import java.io.BufferedWriter
import java.text.SimpleDateFormat
import java.util.Date
import java.util.Calendar
import kotlin.Exception


class BOJCrawler(private val problem: String, private val doubleCheck: Boolean) {
    private val webDriverID = "webdriver.edge.driver"
    private val webDriverPath = "웹드라이버의 경로"
    //private val webDriverPath = "/Users/사용자폴더/Downloads/edgedriver_mac64/msedgedriver"
    private val options: EdgeOptions = EdgeOptions()
    private val driver: EdgeDriver by lazy { EdgeDriver(options) }

    init {
        println("init. Crawler initializing...")
        System.setProperty(webDriverID, webDriverPath)
        options.addArguments("headless")
        options.addArguments("window-size=1920x1080")
        options.addArguments("disable-gpu")
        options.addArguments("disable-infobars")
        options.addArguments("--disable-extensions")
        options.addArguments("--disable-default-apps")

        getProblem()
    }

    private fun getProblem() {
        val url = "https://www.acmicpc.net/problem/"
        println("Crawling from ${url}${problem}")


        try {
            println("Loading Edge driver...")
            driver.get("${url}${problem}")
        } catch (e: WebDriverException) {
            println(e)
        }
    }

    private fun makeMDFrontMatters(): String {
        println("Building markdown file front matters...")
        val strArr = mutableListOf<String>()

        //begin frontmatters
        strArr.add("---")

        //title
        strArr.add(
            let {
                val title = driver
                    .title
                    .split("번:")
                    .map { it.trim() }

                "title: \"[백준] ${title[1]} (${title[0]})(kotlin)\""
            }
        )

        //excerpt
        sstrArr.add(
            let {
                val description = driver
                    .findElement(By.xpath("/html/body/div[2]/div[2]/div[3]/div[5]/div[1]/section/div[2]"))
                    .text
                    .split(".")
                    .let {
                        var text = it[0]
                        if (it.size != 1 && it[0] == "") {
                            for (i in text.indices) {
                                if (it[i] == "") continue
                                else {
                                    text = it[i]
                                    break
                                }
                            }
                        }

                        text
                    }
                    .replace("\"", "\\\"")
                    .replace("\n", " ") + "."

                "excerpt: \"$description\""
            }
        )

        //categories
        strArr.add("categories:\n- boj")

        //tags
        strArr.add("tags:\n- algorithm\n- kotlin")

        //last_modified_at
        strArr.add(
            let {
                val currentDateTime: Date = Calendar.getInstance().time
                val dateFormat: SimpleDateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss+09:00")

                "last_modified_at: ${dateFormat.format(currentDateTime)}"
            }
        )

        //end of frontmatters
        strArr.add("---")
        return strArr.joinToString("\n", "", "\n")
    }

    private fun makeMDBody(): String {
        println("Building markdown body...")
        val strArr = mutableListOf<String>()

        fun br3() = strArr.add("\n\n\n")
        fun br2() = strArr.add("\n\n")
        fun br1() = strArr.add("\n")

        //문제 설명
        br1()
        strArr.add("### 문제 설명")
        strArr.add("[백준 ${problem}번 문제 링크](https://www.acmicpc.net/problem/${problem}#description)")
        br3()

        //문제 입출력 설명
        val problem_input = driver.findElement(By.id("problem_input"))
            .findElements(By.tagName("p"))
            .map {
                it
                    .text
                    .replace("\n", " ")
                    .replace("-", "\\-")
            }
        val problem_output = driver.findElement(By.id("problem_output"))
            .findElements(By.tagName("p"))
            .map {
                it
                    .text
                    .replace("\n", " ")
                    .replace("-", "\\-")
            }
        strArr.add("### 입력 및 출력")
        if (problem_input[0] != "") {
            strArr.add("#### >> 입력")
            if (problem_input.size == 1) {
                strArr.addAll(problem_input)
            } else {
                strArr.add(problem_input.joinToString("\n", "* "))
            }
            br2()
        }
        if (problem_output[0] != "") {
            strArr.add("#### >> 출력")
            if (problem_output.size == 1) {
                strArr.addAll(problem_output)
            } else {
                strArr.add(problem_output.joinToString("\n", "* "))
            }
            br2()
        }
        br1()

        //문제 제한 설명
        val problem_limit = driver.findElement(By.id("problem_limit")).text
        if (problem_limit != "") {
            strArr.add("### 제한 사항")
            br1()
            strArr.add(problem_limit.replace("-", "\\-"))
            br3()
        }

        //문제 예제 입출력
        var sample_input_count = 1
        while (true) {
            try {
                driver.findElement(By.id("sample-input-${sample_input_count}"))
                sample_input_count++
            } catch (e: WebDriverException) {
                sample_input_count--
                break
            }
        }

        if (sample_input_count != 0) {
            strArr.add("### 예제 입출력")
            br1()
            strArr.add("|입력|출력|")
            strArr.add("|-----|------|")
            for (count in 1..sample_input_count) {
                val input = driver.findElement(By.id("sample-input-$count"))
                    .text
                    .let {
                        it.replace("\n", "<br>")
                    }
                    .replace("-", "\\-")
                val output = driver.findElement(By.id("sample-output-$count"))
                    .text
                    .let {
                        it.replace("\n", "<br>")
                    }
                    .replace("-", "\\-")
                strArr.add("|$input|$output|")
            }
            br3()
        }

        //문제 힌트
        val problem_hint = driver.findElement(By.id("problem_hint")).text
        if (problem_hint != "") {
            strArr.add("### 문제 힌트")
            br1()
            strArr.add(problem_hint.replace("-", "\\-"))
            br3()
        }

        driver.close()
        return strArr.joinToString("\n", "", "\n")
    }

    fun makeMDFile() {
        val currentDateTime: Date = Calendar.getInstance().time
        val dateFormat: SimpleDateFormat = SimpleDateFormat("yyyy-MM-dd")
        val path = "마크다운 파일이 저장될 폴더 위치"
        //val path = "/Users/사용자폴더/Documents/GIT_youjourney.github.io/_posts/"
        val fileName = "${dateFormat.format(currentDateTime)}-BOJ${problem}.md"
        println("Making ${fileName}...")

        try {
            val fw = FileWriter(path + fileName)
            val bw = BufferedWriter(fw)

            bw.write(makeMDFrontMatters() + makeMDBody() + loadKtSource())
            bw.close()
            fw.close()
        } catch (e: Exception) {
            println("Can't write the ${dateFormat.format(currentDateTime)}-${problem}.md")
            e.printStackTrace()
        } finally {
            println("$fileName is made successfully!")
            //println("File is written at : /Users/사용자명/Documents/GIT_youjourney.github.io/_posts/${dateFormat.format(currentDateTime)}-${problem}.md")
        }

        if (doubleCheck) {
            println("Double Check is enabled. Checking now.")
            println("======================================")
            try {
                val file = File(path + fileName)
                val fr = FileReader(file)
                val br = BufferedReader(fr)

                br.forEachLine { println(it) }
                br.close()
                fr.close()
            } catch (e: Exception) {
                e.printStackTrace()
            }
        }
    }
}

fun main(args: Array<String>) {
    val doubleCheck: Boolean = args[1]
        .let {
            when (it.toLowerCase()) {
                "true" -> true
                else -> false
            }
        }
    val crawler: BOJCrawler = BOJCrawler(args[0], doubleCheck)

    crawler.makeMDFile()
}

참고

[백준] 호텔 (1106)

세계적인 호텔인 형택 호텔의 사장인 김형택은 이번에 수입을 조금 늘리기 위해서 홍보를 하려고 한다.