Need correct MySQL query to list nearby posts

Unfortunately i can’t help you with that query. Maybe you can make things easier to create an extra custom table to store post locations in.

CREATE TABLE wp_post_locations (
	id BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT,
	post_id BIGINT(20) UNSIGNED DEFAULT NULL,
	lat FLOAT( 10, 6 ) NOT NULL,
	lng FLOAT( 10, 6 ) NOT NULL,
	PRIMARY KEY  (id),
	UNIQUE KEY post_id (post_id)
)

You can then use the ‘posts_clauses’ filter to join this table.

/**
 * Posts clauses
 *
 * http://codex.wordpress.org/WordPress_Query_Vars
 * http://codex.wordpress.org/Custom_Queries
 *
 * @param array $pieces
 * @param WP_Query $query
 * @return string
 */
function custom_locations_posts_clauses( $pieces, $query ) {
	global $wpdb;

	// Fields
	$fields = ",
		location.id AS location_id,
		location.lat AS location_lat,
		location.lng AS location_lng
	";

	// Join
	$join = "
		LEFT JOIN
			$wpdb->post_locations AS location
				ON $wpdb->posts.ID = location.post_id
	";

	// Location
	$latitude  = $query->get( 'latitude' );
	$longitude = $query->get( 'longitude' );

	if ( ! empty( $latitude ) && ! empty( $longitude ) ) {
		$radius = 6371; // KM

		$latitude1 = $wpdb->prepare( "%f", $latitude );
		$latitude2 = 'location.lat';
		$longitude1 =  $wpdb->prepare( "%f", $longitude );
		$longitude2 = 'location.lng';

		$d = "$radius * ACOS( COS( RADIANS( $latitude1 ) ) * COS( RADIANS( $latitude2 ) ) * COS( RADIANS( $longitude1 ) - RADIANS( $longitude2 ) ) + SIN( RADIANS( $latitude1 ) ) * SIN( RADIANS( $latitude2 ) ) )";

		$fields .= ",
			$d AS distance
		";
	}

	$pieces['fields']  .= $fields;
	$pieces['join']    .= $join;

	return $pieces;
}

add_filter( 'posts_clauses', 'custom_locations_posts_clauses', 10, 2 );

You can now use a query like this:

$query = new WP_Query( array(
	'latitude'  => 53,
	'longitude' => 9,
) );

Good luck.